2024-12-03 14:13:31,824 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 14:13:31,836 main DEBUG Took 0.010226 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 14:13:31,837 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 14:13:31,837 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 14:13:31,838 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 14:13:31,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,844 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 14:13:31,856 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,857 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,858 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,859 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,860 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,860 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,861 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,861 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,861 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,862 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,862 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,863 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,863 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,864 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,864 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,864 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:13:31,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,865 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 14:13:31,866 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:13:31,868 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 14:13:31,869 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 14:13:31,869 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 14:13:31,870 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 14:13:31,871 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 14:13:31,878 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 14:13:31,880 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 14:13:31,881 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 14:13:31,882 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 14:13:31,882 main DEBUG createAppenders(={Console}) 2024-12-03 14:13:31,883 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 14:13:31,883 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 14:13:31,883 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 14:13:31,883 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 14:13:31,884 main DEBUG OutputStream closed 2024-12-03 14:13:31,884 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 14:13:31,884 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 14:13:31,884 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 14:13:31,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 14:13:31,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 14:13:31,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 14:13:31,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 14:13:31,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 14:13:31,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 14:13:31,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 14:13:31,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 14:13:31,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 14:13:31,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 14:13:31,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 14:13:31,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 14:13:31,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 14:13:31,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 14:13:31,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 14:13:31,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 14:13:31,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 14:13:31,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 14:13:31,955 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 14:13:31,955 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 14:13:31,955 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 14:13:31,956 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T14:13:32,156 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984 2024-12-03 14:13:32,159 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 14:13:32,160 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T14:13:32,167 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T14:13:32,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T14:13:32,197 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398, deleteOnExit=true 2024-12-03T14:13:32,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T14:13:32,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/test.cache.data in system properties and HBase conf 2024-12-03T14:13:32,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T14:13:32,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.log.dir in system properties and HBase conf 2024-12-03T14:13:32,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T14:13:32,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T14:13:32,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T14:13:32,275 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T14:13:32,351 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T14:13:32,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:13:32,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:13:32,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T14:13:32,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:13:32,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T14:13:32,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T14:13:32,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:13:32,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:13:32,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T14:13:32,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/nfs.dump.dir in system properties and HBase conf 2024-12-03T14:13:32,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/java.io.tmpdir in system properties and HBase conf 2024-12-03T14:13:32,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:13:32,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T14:13:32,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T14:13:33,359 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T14:13:33,423 INFO [Time-limited test {}] log.Log(170): Logging initialized @2188ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T14:13:33,482 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:13:33,532 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:13:33,549 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:13:33,549 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:13:33,551 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T14:13:33,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:13:33,565 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:13:33,566 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:13:33,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/java.io.tmpdir/jetty-localhost-35195-hadoop-hdfs-3_4_1-tests_jar-_-any-8973489526093375806/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:13:33,726 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35195} 2024-12-03T14:13:33,726 INFO [Time-limited test {}] server.Server(415): Started @2492ms 2024-12-03T14:13:34,255 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:13:34,261 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:13:34,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:13:34,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:13:34,263 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T14:13:34,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:13:34,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:13:34,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/java.io.tmpdir/jetty-localhost-46515-hadoop-hdfs-3_4_1-tests_jar-_-any-13346192560736491480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:13:34,357 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:46515} 2024-12-03T14:13:34,357 INFO [Time-limited test {}] server.Server(415): Started @3123ms 2024-12-03T14:13:34,401 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T14:13:35,374 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data2/current/BP-1954622350-172.17.0.2-1733235212894/current, will proceed with Du for space computation calculation, 2024-12-03T14:13:35,374 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data1/current/BP-1954622350-172.17.0.2-1733235212894/current, will proceed with Du for space computation calculation, 2024-12-03T14:13:35,404 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T14:13:35,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x75e88ce93ee43d22 with lease ID 0x42be62a20cb0b041: Processing first storage report for DS-c492f942-213c-452d-b099-aeccf44f3fe3 from datanode DatanodeRegistration(127.0.0.1:35659, datanodeUuid=7a205582-7199-4e8f-8cff-3b983e70ba6e, infoPort=36053, infoSecurePort=0, ipcPort=40981, storageInfo=lv=-57;cid=testClusterID;nsid=906950719;c=1733235212894) 2024-12-03T14:13:35,444 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x75e88ce93ee43d22 with lease ID 0x42be62a20cb0b041: from storage DS-c492f942-213c-452d-b099-aeccf44f3fe3 node DatanodeRegistration(127.0.0.1:35659, datanodeUuid=7a205582-7199-4e8f-8cff-3b983e70ba6e, infoPort=36053, infoSecurePort=0, ipcPort=40981, storageInfo=lv=-57;cid=testClusterID;nsid=906950719;c=1733235212894), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T14:13:35,444 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x75e88ce93ee43d22 with lease ID 0x42be62a20cb0b041: Processing first storage report for DS-4ace96ae-6b0e-4f8f-9bbe-8e8aa2071c64 from datanode DatanodeRegistration(127.0.0.1:35659, datanodeUuid=7a205582-7199-4e8f-8cff-3b983e70ba6e, infoPort=36053, infoSecurePort=0, ipcPort=40981, storageInfo=lv=-57;cid=testClusterID;nsid=906950719;c=1733235212894) 2024-12-03T14:13:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x75e88ce93ee43d22 with lease ID 0x42be62a20cb0b041: from storage DS-4ace96ae-6b0e-4f8f-9bbe-8e8aa2071c64 node DatanodeRegistration(127.0.0.1:35659, datanodeUuid=7a205582-7199-4e8f-8cff-3b983e70ba6e, infoPort=36053, infoSecurePort=0, ipcPort=40981, storageInfo=lv=-57;cid=testClusterID;nsid=906950719;c=1733235212894), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T14:13:35,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984 2024-12-03T14:13:35,526 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/zookeeper_0, clientPort=58521, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T14:13:35,535 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58521 2024-12-03T14:13:35,545 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:35,548 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:35,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741825_1001 (size=7) 2024-12-03T14:13:36,148 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f with version=8 2024-12-03T14:13:36,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/hbase-staging 2024-12-03T14:13:36,221 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T14:13:36,406 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3920e6a7c3b5:0 server-side Connection retries=45 2024-12-03T14:13:36,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:36,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:36,418 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:13:36,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:36,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:13:36,525 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T14:13:36,572 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T14:13:36,579 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T14:13:36,582 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:13:36,604 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4013 (auto-detected) 2024-12-03T14:13:36,605 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T14:13:36,621 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42837 2024-12-03T14:13:36,640 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42837 connecting to ZooKeeper ensemble=127.0.0.1:58521 2024-12-03T14:13:36,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428370x0, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:13:36,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42837-0x1019b8a88400000 connected 2024-12-03T14:13:36,891 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:36,895 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:36,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:13:36,912 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f, hbase.cluster.distributed=false 2024-12-03T14:13:36,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:13:36,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42837 2024-12-03T14:13:36,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42837 2024-12-03T14:13:36,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42837 2024-12-03T14:13:36,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42837 2024-12-03T14:13:36,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42837 2024-12-03T14:13:37,041 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3920e6a7c3b5:0 server-side Connection retries=45 2024-12-03T14:13:37,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:37,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:37,043 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:13:37,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:13:37,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:13:37,047 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T14:13:37,049 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:13:37,050 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42177 2024-12-03T14:13:37,052 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42177 connecting to ZooKeeper ensemble=127.0.0.1:58521 2024-12-03T14:13:37,054 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:37,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:37,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421770x0, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:13:37,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421770x0, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:13:37,067 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42177-0x1019b8a88400001 connected 2024-12-03T14:13:37,071 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T14:13:37,078 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T14:13:37,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T14:13:37,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:13:37,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42177 2024-12-03T14:13:37,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42177 2024-12-03T14:13:37,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42177 2024-12-03T14:13:37,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42177 2024-12-03T14:13:37,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42177 2024-12-03T14:13:37,106 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3920e6a7c3b5:42837 2024-12-03T14:13:37,107 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:37,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:13:37,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:13:37,121 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:37,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:37,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T14:13:37,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:37,151 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T14:13:37,152 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3920e6a7c3b5,42837,1733235216263 from backup master directory 2024-12-03T14:13:37,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:37,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:13:37,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:13:37,162 WARN [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:13:37,162 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:37,164 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T14:13:37,165 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T14:13:37,213 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/hbase.id] with ID: d246842c-db5a-49e0-a354-4fa4c8943284 2024-12-03T14:13:37,213 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/.tmp/hbase.id 2024-12-03T14:13:37,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741826_1002 (size=42) 2024-12-03T14:13:37,627 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/.tmp/hbase.id]:[hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/hbase.id] 2024-12-03T14:13:37,678 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:13:37,682 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T14:13:37,699 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T14:13:37,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:37,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:37,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741827_1003 (size=196) 2024-12-03T14:13:38,189 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:13:38,192 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T14:13:38,204 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:38,208 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:13:38,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741828_1004 (size=1189) 2024-12-03T14:13:38,657 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store 2024-12-03T14:13:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741829_1005 (size=34) 2024-12-03T14:13:39,086 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T14:13:39,089 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:39,090 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:13:39,090 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:13:39,091 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:13:39,092 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:13:39,092 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:13:39,092 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:13:39,093 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733235219090Disabling compacts and flushes for region at 1733235219090Disabling writes for close at 1733235219092 (+2 ms)Writing region close event to WAL at 1733235219092Closed at 1733235219092 2024-12-03T14:13:39,095 WARN [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/.initializing 2024-12-03T14:13:39,095 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/WALs/3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:39,102 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:13:39,115 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3920e6a7c3b5%2C42837%2C1733235216263, suffix=, logDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/WALs/3920e6a7c3b5,42837,1733235216263, archiveDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/oldWALs, maxLogs=10 2024-12-03T14:13:39,134 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/WALs/3920e6a7c3b5,42837,1733235216263/3920e6a7c3b5%2C42837%2C1733235216263.1733235219119, exclude list is [], retry=0 2024-12-03T14:13:39,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-c492f942-213c-452d-b099-aeccf44f3fe3,DISK] 2024-12-03T14:13:39,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T14:13:39,182 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/WALs/3920e6a7c3b5,42837,1733235216263/3920e6a7c3b5%2C42837%2C1733235216263.1733235219119 2024-12-03T14:13:39,183 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36053:36053)] 2024-12-03T14:13:39,183 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:13:39,184 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:39,187 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,188 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T14:13:39,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:39,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:39,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T14:13:39,247 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:39,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:39,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T14:13:39,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:39,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:39,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T14:13:39,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:39,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:39,256 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,260 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,261 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,266 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,267 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,271 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:13:39,276 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:13:39,280 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:13:39,281 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74946958, jitterRate=0.11679670214653015}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:13:39,285 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733235219198Initializing all the Stores at 1733235219200 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235219200Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235219201 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235219201Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235219201Cleaning up temporary data from old regions at 1733235219267 (+66 ms)Region opened successfully at 1733235219285 (+18 ms) 2024-12-03T14:13:39,286 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T14:13:39,313 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d556f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3920e6a7c3b5/172.17.0.2:0 2024-12-03T14:13:39,338 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T14:13:39,348 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T14:13:39,349 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T14:13:39,351 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T14:13:39,352 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T14:13:39,356 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T14:13:39,356 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T14:13:39,382 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T14:13:39,391 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T14:13:39,445 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T14:13:39,449 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T14:13:39,452 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T14:13:39,465 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T14:13:39,467 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T14:13:39,472 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T14:13:39,476 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T14:13:39,478 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T14:13:39,487 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T14:13:39,507 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T14:13:39,518 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T14:13:39,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:13:39,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:13:39,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,534 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3920e6a7c3b5,42837,1733235216263, sessionid=0x1019b8a88400000, setting cluster-up flag (Was=false) 2024-12-03T14:13:39,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,593 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T14:13:39,596 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:39,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:39,655 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T14:13:39,658 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:39,667 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T14:13:39,699 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(746): ClusterId : d246842c-db5a-49e0-a354-4fa4c8943284 2024-12-03T14:13:39,701 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T14:13:39,710 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T14:13:39,710 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T14:13:39,720 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T14:13:39,720 DEBUG [RS:0;3920e6a7c3b5:42177 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f38b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3920e6a7c3b5/172.17.0.2:0 2024-12-03T14:13:39,734 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3920e6a7c3b5:42177 2024-12-03T14:13:39,737 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T14:13:39,737 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T14:13:39,737 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T14:13:39,738 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T14:13:39,739 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(2659): reportForDuty to master=3920e6a7c3b5,42837,1733235216263 with port=42177, startcode=1733235217010 2024-12-03T14:13:39,747 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T14:13:39,748 DEBUG [RS:0;3920e6a7c3b5:42177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T14:13:39,753 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T14:13:39,758 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3920e6a7c3b5,42837,1733235216263 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T14:13:39,766 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3920e6a7c3b5:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:13:39,766 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3920e6a7c3b5:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3920e6a7c3b5:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3920e6a7c3b5:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3920e6a7c3b5:0, corePoolSize=10, maxPoolSize=10 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3920e6a7c3b5:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:13:39,767 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:39,772 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:13:39,772 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733235249772 2024-12-03T14:13:39,772 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T14:13:39,774 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T14:13:39,775 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T14:13:39,778 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:39,778 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T14:13:39,780 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T14:13:39,780 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T14:13:39,781 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T14:13:39,782 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T14:13:39,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34519, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T14:13:39,783 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:39,787 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T14:13:39,789 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T14:13:39,789 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T14:13:39,791 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42837 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:39,792 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T14:13:39,793 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T14:13:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741831_1007 (size=1321) 2024-12-03T14:13:39,797 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.large.0-1733235219794,5,FailOnTimeoutGroup] 2024-12-03T14:13:39,800 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.small.0-1733235219797,5,FailOnTimeoutGroup] 2024-12-03T14:13:39,800 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:39,800 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T14:13:39,802 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:39,802 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:39,823 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T14:13:39,823 WARN [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T14:13:39,925 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(2659): reportForDuty to master=3920e6a7c3b5,42837,1733235216263 with port=42177, startcode=1733235217010 2024-12-03T14:13:39,928 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42837 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:39,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42837 {}] master.ServerManager(517): Registering regionserver=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:39,939 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:13:39,939 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40805 2024-12-03T14:13:39,939 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T14:13:39,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:13:39,951 DEBUG [RS:0;3920e6a7c3b5:42177 {}] zookeeper.ZKUtil(111): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:39,951 WARN [RS:0;3920e6a7c3b5:42177 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:13:39,951 INFO [RS:0;3920e6a7c3b5:42177 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:13:39,952 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:39,953 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3920e6a7c3b5,42177,1733235217010] 2024-12-03T14:13:39,978 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T14:13:39,989 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T14:13:39,994 INFO [RS:0;3920e6a7c3b5:42177 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:13:39,994 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:39,995 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T14:13:39,999 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T14:13:40,000 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,000 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,001 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,001 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,001 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,001 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,001 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3920e6a7c3b5:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:13:40,002 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,002 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,002 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,002 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,003 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,003 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3920e6a7c3b5:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:13:40,003 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3920e6a7c3b5:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:13:40,003 DEBUG [RS:0;3920e6a7c3b5:42177 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:13:40,004 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,004 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,005 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,005 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,005 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,005 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42177,1733235217010-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:13:40,028 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T14:13:40,029 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42177,1733235217010-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,030 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,030 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.Replication(171): 3920e6a7c3b5,42177,1733235217010 started 2024-12-03T14:13:40,045 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:40,045 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1482): Serving as 3920e6a7c3b5,42177,1733235217010, RpcServer on 3920e6a7c3b5/172.17.0.2:42177, sessionid=0x1019b8a88400001 2024-12-03T14:13:40,046 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T14:13:40,046 DEBUG [RS:0;3920e6a7c3b5:42177 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:40,046 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3920e6a7c3b5,42177,1733235217010' 2024-12-03T14:13:40,047 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T14:13:40,048 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T14:13:40,048 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T14:13:40,048 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T14:13:40,049 DEBUG [RS:0;3920e6a7c3b5:42177 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:40,049 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3920e6a7c3b5,42177,1733235217010' 2024-12-03T14:13:40,049 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T14:13:40,049 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T14:13:40,050 DEBUG [RS:0;3920e6a7c3b5:42177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T14:13:40,050 INFO [RS:0;3920e6a7c3b5:42177 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T14:13:40,050 INFO [RS:0;3920e6a7c3b5:42177 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T14:13:40,158 INFO [RS:0;3920e6a7c3b5:42177 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:13:40,163 INFO [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3920e6a7c3b5%2C42177%2C1733235217010, suffix=, logDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010, archiveDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/oldWALs, maxLogs=32 2024-12-03T14:13:40,180 DEBUG [RS:0;3920e6a7c3b5:42177 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010/3920e6a7c3b5%2C42177%2C1733235217010.1733235220166, exclude list is [], retry=0 2024-12-03T14:13:40,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-c492f942-213c-452d-b099-aeccf44f3fe3,DISK] 2024-12-03T14:13:40,188 INFO [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010/3920e6a7c3b5%2C42177%2C1733235217010.1733235220166 2024-12-03T14:13:40,189 DEBUG [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36053:36053)] 2024-12-03T14:13:40,197 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T14:13:40,197 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:13:40,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741833_1009 (size=32) 2024-12-03T14:13:40,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:40,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:13:40,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:13:40,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:40,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:40,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:13:40,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:13:40,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:40,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:40,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:13:40,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:13:40,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:40,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:40,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:13:40,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:13:40,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:40,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:40,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:13:40,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740 2024-12-03T14:13:40,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740 2024-12-03T14:13:40,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:13:40,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:13:40,642 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:13:40,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:13:40,648 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:13:40,649 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60926114, jitterRate=-0.0921301543712616}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:13:40,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733235220612Initializing all the Stores at 1733235220617 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235220617Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235220618 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235220618Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235220618Cleaning up temporary data from old regions at 1733235220641 (+23 ms)Region opened successfully at 1733235220650 (+9 ms) 2024-12-03T14:13:40,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:13:40,651 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:13:40,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:13:40,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:13:40,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:13:40,652 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:13:40,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733235220650Disabling compacts and flushes for region at 1733235220650Disabling writes for close at 1733235220651 (+1 ms)Writing region close event to WAL at 1733235220652 (+1 ms)Closed at 1733235220652 2024-12-03T14:13:40,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:13:40,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T14:13:40,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T14:13:40,668 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:13:40,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T14:13:40,827 DEBUG [3920e6a7c3b5:42837 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T14:13:40,840 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:40,846 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3920e6a7c3b5,42177,1733235217010, state=OPENING 2024-12-03T14:13:40,909 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T14:13:40,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:40,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:13:40,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:13:40,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:13:40,922 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:13:40,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:13:41,103 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:13:41,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32879, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:13:41,116 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T14:13:41,116 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:13:41,117 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T14:13:41,120 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3920e6a7c3b5%2C42177%2C1733235217010.meta, suffix=.meta, logDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010, archiveDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/oldWALs, maxLogs=32 2024-12-03T14:13:41,135 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010/3920e6a7c3b5%2C42177%2C1733235217010.meta.1733235221122.meta, exclude list is [], retry=0 2024-12-03T14:13:41,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-c492f942-213c-452d-b099-aeccf44f3fe3,DISK] 2024-12-03T14:13:41,141 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/WALs/3920e6a7c3b5,42177,1733235217010/3920e6a7c3b5%2C42177%2C1733235217010.meta.1733235221122.meta 2024-12-03T14:13:41,142 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36053:36053)] 2024-12-03T14:13:41,142 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:13:41,144 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T14:13:41,147 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T14:13:41,151 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T14:13:41,155 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T14:13:41,155 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:41,155 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T14:13:41,156 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T14:13:41,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:13:41,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:13:41,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:41,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:41,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:13:41,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:13:41,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:41,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:41,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:13:41,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:13:41,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:41,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:41,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:13:41,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:13:41,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:41,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:13:41,168 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:13:41,170 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740 2024-12-03T14:13:41,173 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740 2024-12-03T14:13:41,175 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:13:41,175 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:13:41,176 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:13:41,179 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:13:41,181 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70521294, jitterRate=0.05084916949272156}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:13:41,181 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T14:13:41,183 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733235221156Writing region info on filesystem at 1733235221156Initializing all the Stores at 1733235221158 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235221158Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235221158Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235221158Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733235221158Cleaning up temporary data from old regions at 1733235221175 (+17 ms)Running coprocessor post-open hooks at 1733235221181 (+6 ms)Region opened successfully at 1733235221183 (+2 ms) 2024-12-03T14:13:41,188 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733235221096 2024-12-03T14:13:41,198 DEBUG [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T14:13:41,199 INFO [RS_OPEN_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T14:13:41,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:41,202 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3920e6a7c3b5,42177,1733235217010, state=OPEN 2024-12-03T14:13:41,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:13:41,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:13:41,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:13:41,286 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:13:41,286 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:41,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T14:13:41,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3920e6a7c3b5,42177,1733235217010 in 363 msec 2024-12-03T14:13:41,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T14:13:41,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 635 msec 2024-12-03T14:13:41,302 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:13:41,302 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T14:13:41,320 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:41,321 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:41,337 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:41,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48137, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:41,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6630 sec 2024-12-03T14:13:41,359 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733235221359, completionTime=-1 2024-12-03T14:13:41,362 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T14:13:41,362 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T14:13:41,385 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T14:13:41,385 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733235281385 2024-12-03T14:13:41,385 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733235341385 2024-12-03T14:13:41,385 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-03T14:13:41,387 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,387 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,387 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,388 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3920e6a7c3b5:42837, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,388 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,389 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,393 DEBUG [master/3920e6a7c3b5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T14:13:41,413 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.251sec 2024-12-03T14:13:41,414 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T14:13:41,415 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T14:13:41,415 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T14:13:41,416 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T14:13:41,416 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T14:13:41,417 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:13:41,418 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T14:13:41,425 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T14:13:41,426 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T14:13:41,426 INFO [master/3920e6a7c3b5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3920e6a7c3b5,42837,1733235216263-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:13:41,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:41,518 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T14:13:41,519 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T14:13:41,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:41,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:41,529 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:41,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:41,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a11d50b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:41,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:41,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:41,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:41,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:41,543 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:41,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:41,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:41,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:41,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=10872 2024-12-03T14:13:41,614 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:13:41,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e32371d 2024-12-03T14:13:41,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:13:41,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:13:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:13:41,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:13:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:13:41,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:13:41,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T14:13:41,643 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:41,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:13:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:41,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741835_1011 (size=1061) 2024-12-03T14:13:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:42,061 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:13:42,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741836_1012 (size=53) 2024-12-03T14:13:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:42,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:42,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8a9798ce308c36fb733235c6d0f2266b, disabling compactions & flushes 2024-12-03T14:13:42,473 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. after waiting 0 ms 2024-12-03T14:13:42,473 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,473 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,474 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a9798ce308c36fb733235c6d0f2266b: Waiting for close lock at 1733235222473Disabling compacts and flushes for region at 1733235222473Disabling writes for close at 1733235222473Writing region close event to WAL at 1733235222473Closed at 1733235222473 2024-12-03T14:13:42,476 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:13:42,482 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235222476"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235222476"}]},"ts":"1733235222476"} 2024-12-03T14:13:42,487 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:13:42,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:13:42,492 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235222489"}]},"ts":"1733235222489"} 2024-12-03T14:13:42,496 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:13:42,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, ASSIGN}] 2024-12-03T14:13:42,500 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, ASSIGN 2024-12-03T14:13:42,501 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:13:42,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a9798ce308c36fb733235c6d0f2266b, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:42,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, ASSIGN because future has completed 2024-12-03T14:13:42,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:13:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:42,829 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,829 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:13:42,830 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,830 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:13:42,831 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,831 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,834 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,837 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:13:42,838 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a9798ce308c36fb733235c6d0f2266b columnFamilyName A 2024-12-03T14:13:42,838 DEBUG [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:42,839 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(327): Store=8a9798ce308c36fb733235c6d0f2266b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:42,840 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,843 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:13:42,844 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a9798ce308c36fb733235c6d0f2266b columnFamilyName B 2024-12-03T14:13:42,844 DEBUG [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:42,846 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(327): Store=8a9798ce308c36fb733235c6d0f2266b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:42,846 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,850 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:13:42,850 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a9798ce308c36fb733235c6d0f2266b columnFamilyName C 2024-12-03T14:13:42,851 DEBUG [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:13:42,853 INFO [StoreOpener-8a9798ce308c36fb733235c6d0f2266b-1 {}] regionserver.HStore(327): Store=8a9798ce308c36fb733235c6d0f2266b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:13:42,854 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,854 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,857 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,858 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,863 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,863 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,865 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:13:42,870 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,877 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:13:42,879 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8a9798ce308c36fb733235c6d0f2266b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61108474, jitterRate=-0.08941277861595154}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:13:42,879 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:42,880 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8a9798ce308c36fb733235c6d0f2266b: Running coprocessor pre-open hook at 1733235222831Writing region info on filesystem at 1733235222831Initializing all the Stores at 1733235222833 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235222833Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235222833Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235222833Cleaning up temporary data from old regions at 1733235222863 (+30 ms)Running coprocessor post-open hooks at 1733235222879 (+16 ms)Region opened successfully at 1733235222880 (+1 ms) 2024-12-03T14:13:42,883 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., pid=6, masterSystemTime=1733235222818 2024-12-03T14:13:42,887 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,888 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:42,889 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a9798ce308c36fb733235c6d0f2266b, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:42,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:13:42,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T14:13:42,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 in 233 msec 2024-12-03T14:13:42,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T14:13:42,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, ASSIGN in 404 msec 2024-12-03T14:13:42,910 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:13:42,910 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235222910"}]},"ts":"1733235222910"} 2024-12-03T14:13:42,913 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:13:42,915 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:13:42,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2800 sec 2024-12-03T14:13:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:13:43,798 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:43,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470128a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,816 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3858e840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,819 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:13:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:13:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e7e947e 2024-12-03T14:13:43,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:13:43,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:13:43,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,837 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a8011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1380f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,844 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6d3ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27cf2c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,851 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fadb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff4b0a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fa4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,865 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167ce6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23efbfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,872 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5385c31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5612ff47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cba810e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bdb462e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32afd8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:13:43,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:13:43,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:13:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:13:43,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:13:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3714a06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:13:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:13:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:13:43,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:13:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4689148e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:13:43,896 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,896 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,896 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,896 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,897 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,899 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,905 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,906 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T14:13:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:43,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,915 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,918 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:43,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,925 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:13:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:13:43,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:13:43,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:13:43,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:13:43,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:13:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:43,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:43,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:43,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:43,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:44,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/1d76f19db3ef4bdf9026df6673ac0ffa is 50, key is test_row_0/A:col10/1733235223963/Put/seqid=0 2024-12-03T14:13:44,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741837_1013 (size=12001) 2024-12-03T14:13:44,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235224087, max timestamp=1733235224104Num uniques 859; 2024-12-03T14:13:44,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235224087, max timestamp=1733235224104Num uniques 859; 2024-12-03T14:13:44,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:13:44,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:13:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235284094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733235224087, max timestamp=1733235224102Num uniques 859; 2024-12-03T14:13:44,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T14:13:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235284179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235284182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235284191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235284191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235284237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235284297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235284298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235284306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235284307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235284447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/1d76f19db3ef4bdf9026df6673ac0ffa 2024-12-03T14:13:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235284518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235284518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235284519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235284519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:44,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/6c38c92714894ec09fdb6a68704931a2 is 50, key is test_row_0/B:col10/1733235223963/Put/seqid=0 2024-12-03T14:13:44,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741838_1014 (size=12001) 2024-12-03T14:13:44,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/6c38c92714894ec09fdb6a68704931a2 2024-12-03T14:13:44,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7f8db4d1d6c44b1a8db51ba384bbf173 is 50, key is test_row_0/C:col10/1733235223963/Put/seqid=0 2024-12-03T14:13:44,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741839_1015 (size=12001) 2024-12-03T14:13:44,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7f8db4d1d6c44b1a8db51ba384bbf173 2024-12-03T14:13:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/1d76f19db3ef4bdf9026df6673ac0ffa as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa 2024-12-03T14:13:44,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:13:44,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/6c38c92714894ec09fdb6a68704931a2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2 2024-12-03T14:13:44,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:13:44,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7f8db4d1d6c44b1a8db51ba384bbf173 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173 2024-12-03T14:13:44,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:13:44,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.78 KB/136995 for 8a9798ce308c36fb733235c6d0f2266b in 824ms, sequenceid=12, compaction requested=false 2024-12-03T14:13:44,803 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T14:13:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:44,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733235224154, max timestamp=1733235224793Num uniques 1018; 2024-12-03T14:13:44,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T14:13:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:44,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=102.48 KB heapSize=291.29 KB 2024-12-03T14:13:44,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:13:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:13:44,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6130d50907424b799d79f6831d553355 is 50, key is test_row_0/A:col10/1733235224184/Put/seqid=0 2024-12-03T14:13:44,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741840_1016 (size=14341) 2024-12-03T14:13:44,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6130d50907424b799d79f6831d553355 2024-12-03T14:13:44,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/d8818edb230248a588d21173c86c67ce is 50, key is test_row_0/B:col10/1733235224184/Put/seqid=0 2024-12-03T14:13:44,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:44,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:44,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741841_1017 (size=12001) 2024-12-03T14:13:44,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/d8818edb230248a588d21173c86c67ce 2024-12-03T14:13:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235284898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235284893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235284917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235284923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/1e6b84f1748944c9abdd843a1bffbe30 is 50, key is test_row_0/C:col10/1733235224184/Put/seqid=0 2024-12-03T14:13:44,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:44,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235284952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:44,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:44,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741842_1018 (size=12001) 2024-12-03T14:13:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235285057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235285067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235285068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235285077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235285087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:45,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235285276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235285287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235285287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235285297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235285297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/1e6b84f1748944c9abdd843a1bffbe30 2024-12-03T14:13:45,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:45,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:45,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:45,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6130d50907424b799d79f6831d553355 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355 2024-12-03T14:13:45,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355, entries=200, sequenceid=50, filesize=14.0 K 2024-12-03T14:13:45,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/d8818edb230248a588d21173c86c67ce as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce 2024-12-03T14:13:45,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T14:13:45,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/1e6b84f1748944c9abdd843a1bffbe30 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30 2024-12-03T14:13:45,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T14:13:45,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.48 KB/104935, heapSize ~291.24 KB/298232, currentSize=80.51 KB/82440 for 8a9798ce308c36fb733235c6d0f2266b in 641ms, sequenceid=50, compaction requested=false 2024-12-03T14:13:45,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:45,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T14:13:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/45862e0707444414b5c985306d87f180 is 50, key is test_row_0/A:col10/1733235224857/Put/seqid=0 2024-12-03T14:13:45,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741843_1019 (size=12001) 2024-12-03T14:13:45,585 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/45862e0707444414b5c985306d87f180 2024-12-03T14:13:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:45,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f7e5c0c1c00d4e9fb5aad86809fb3345 is 50, key is test_row_0/B:col10/1733235224857/Put/seqid=0 2024-12-03T14:13:45,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741844_1020 (size=12001) 2024-12-03T14:13:45,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235285635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235285635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235285675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235285678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235285678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235285788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,973 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T14:13:45,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235285996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235285997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235285997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235286007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235286007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,043 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f7e5c0c1c00d4e9fb5aad86809fb3345 2024-12-03T14:13:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:46,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/69d5043353ec4fa4a0bc548106d4ccfd is 50, key is test_row_0/C:col10/1733235224857/Put/seqid=0 2024-12-03T14:13:46,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741845_1021 (size=12001) 2024-12-03T14:13:46,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235286307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235286307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235286307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235286317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235286317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/69d5043353ec4fa4a0bc548106d4ccfd 2024-12-03T14:13:46,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/45862e0707444414b5c985306d87f180 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180 2024-12-03T14:13:46,519 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T14:13:46,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f7e5c0c1c00d4e9fb5aad86809fb3345 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345 2024-12-03T14:13:46,538 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T14:13:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/69d5043353ec4fa4a0bc548106d4ccfd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd 2024-12-03T14:13:46,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd, entries=150, sequenceid=65, filesize=11.7 K 2024-12-03T14:13:46,562 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 8a9798ce308c36fb733235c6d0f2266b in 1014ms, sequenceid=65, compaction requested=true 2024-12-03T14:13:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T14:13:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T14:13:46,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T14:13:46,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-12-03T14:13:46,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6720 sec 2024-12-03T14:13:46,664 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:13:46,724 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T14:13:46,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T14:13:46,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:13:46,728 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T14:13:46,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T14:13:46,729 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T14:13:46,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T14:13:46,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T14:13:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:46,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:46,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:46,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3de22003861746c2815a99a3186a6ab4 is 50, key is test_row_0/A:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:46,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235286850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235286853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235286856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235286856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235286857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741846_1022 (size=12001) 2024-12-03T14:13:46,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3de22003861746c2815a99a3186a6ab4 2024-12-03T14:13:46,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/31e6b577e3a143a5935c2594e1615809 is 50, key is test_row_0/B:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:46,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741847_1023 (size=12001) 2024-12-03T14:13:46,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/31e6b577e3a143a5935c2594e1615809 2024-12-03T14:13:46,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9ae1f7263ccf4a6cbb778e87977df908 is 50, key is test_row_0/C:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741848_1024 (size=12001) 2024-12-03T14:13:46,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9ae1f7263ccf4a6cbb778e87977df908 2024-12-03T14:13:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235286967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235286967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235286977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235286977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235286977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:46,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:46,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3de22003861746c2815a99a3186a6ab4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4 2024-12-03T14:13:47,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4, entries=150, sequenceid=85, filesize=11.7 K 2024-12-03T14:13:47,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/31e6b577e3a143a5935c2594e1615809 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809 2024-12-03T14:13:47,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809, entries=150, sequenceid=85, filesize=11.7 K 2024-12-03T14:13:47,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9ae1f7263ccf4a6cbb778e87977df908 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908 2024-12-03T14:13:47,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908, entries=150, sequenceid=85, filesize=11.7 K 2024-12-03T14:13:47,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a9798ce308c36fb733235c6d0f2266b in 222ms, sequenceid=85, compaction requested=true 2024-12-03T14:13:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:47,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:47,063 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:47,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:47,065 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:47,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:47,068 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:47,070 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:47,071 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:47,072 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=46.9 K 2024-12-03T14:13:47,076 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c38c92714894ec09fdb6a68704931a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235223917 2024-12-03T14:13:47,077 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:47,077 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:47,077 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:47,077 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d8818edb230248a588d21173c86c67ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235224154 2024-12-03T14:13:47,077 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=49.2 K 2024-12-03T14:13:47,078 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e5c0c1c00d4e9fb5aad86809fb3345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733235224857 2024-12-03T14:13:47,079 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d76f19db3ef4bdf9026df6673ac0ffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235223917 2024-12-03T14:13:47,079 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e6b577e3a143a5935c2594e1615809, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:47,080 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6130d50907424b799d79f6831d553355, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235224154 2024-12-03T14:13:47,081 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45862e0707444414b5c985306d87f180, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733235224857 2024-12-03T14:13:47,082 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de22003861746c2815a99a3186a6ab4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:47,133 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:47,133 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:47,134 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/274ae10ba22e439a89629cad1ffa12ab is 50, key is test_row_0/B:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:47,142 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/4dee1ff4b11f4ac8a2f944086c937bd5 is 50, key is test_row_0/A:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:47,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:13:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:47,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741850_1026 (size=12139) 2024-12-03T14:13:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741849_1025 (size=12139) 2024-12-03T14:13:47,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/03e5c0e631494caeb8f083c9a97692e3 is 50, key is test_row_0/A:col10/1733235226851/Put/seqid=0 2024-12-03T14:13:47,204 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/4dee1ff4b11f4ac8a2f944086c937bd5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/4dee1ff4b11f4ac8a2f944086c937bd5 2024-12-03T14:13:47,204 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/274ae10ba22e439a89629cad1ffa12ab as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/274ae10ba22e439a89629cad1ffa12ab 2024-12-03T14:13:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741851_1027 (size=14341) 2024-12-03T14:13:47,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/03e5c0e631494caeb8f083c9a97692e3 2024-12-03T14:13:47,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cf9b83dc0f774533b2b25955eb31c052 is 50, key is test_row_0/B:col10/1733235226851/Put/seqid=0 2024-12-03T14:13:47,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741852_1028 (size=12001) 2024-12-03T14:13:47,254 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into 4dee1ff4b11f4ac8a2f944086c937bd5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:47,254 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 274ae10ba22e439a89629cad1ffa12ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:47,254 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:47,254 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:47,256 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=12, startTime=1733235227049; duration=0sec 2024-12-03T14:13:47,256 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=12, startTime=1733235227064; duration=0sec 2024-12-03T14:13:47,257 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:47,257 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:47,257 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:47,259 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:47,259 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:47,263 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:47,263 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:47,263 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:47,264 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=46.9 K 2024-12-03T14:13:47,266 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8db4d1d6c44b1a8db51ba384bbf173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235223917 2024-12-03T14:13:47,268 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e6b84f1748944c9abdd843a1bffbe30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235224154 2024-12-03T14:13:47,269 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d5043353ec4fa4a0bc548106d4ccfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733235224857 2024-12-03T14:13:47,270 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae1f7263ccf4a6cbb778e87977df908, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235287276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235287280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235287284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235287288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235287288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,324 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:47,325 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/b5d1da2f1f114d87b689d4ad486c5333 is 50, key is test_row_0/C:col10/1733235225630/Put/seqid=0 2024-12-03T14:13:47,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741853_1029 (size=12139) 2024-12-03T14:13:47,366 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/b5d1da2f1f114d87b689d4ad486c5333 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b5d1da2f1f114d87b689d4ad486c5333 2024-12-03T14:13:47,377 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into b5d1da2f1f114d87b689d4ad486c5333(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:47,377 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:47,378 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=12, startTime=1733235227064; duration=0sec 2024-12-03T14:13:47,378 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:47,378 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:47,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235287397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235287398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235287406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235287416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235287428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235287606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235287606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235287616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235287626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235287646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cf9b83dc0f774533b2b25955eb31c052 2024-12-03T14:13:47,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8d8b8e9b19f8462d8f975296a17d74fb is 50, key is test_row_0/C:col10/1733235226851/Put/seqid=0 2024-12-03T14:13:47,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741854_1030 (size=12001) 2024-12-03T14:13:47,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8d8b8e9b19f8462d8f975296a17d74fb 2024-12-03T14:13:47,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/03e5c0e631494caeb8f083c9a97692e3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3 2024-12-03T14:13:47,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3, entries=200, sequenceid=98, filesize=14.0 K 2024-12-03T14:13:47,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cf9b83dc0f774533b2b25955eb31c052 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052 2024-12-03T14:13:47,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T14:13:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8d8b8e9b19f8462d8f975296a17d74fb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb 2024-12-03T14:13:47,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T14:13:47,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a9798ce308c36fb733235c6d0f2266b in 563ms, sequenceid=98, compaction requested=false 2024-12-03T14:13:47,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:47,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T14:13:47,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:47,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:47,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:47,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:47,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/38b021f8f7384ace8dc33bfe4222953e is 50, key is test_row_0/A:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235287943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235287947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235287947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235287954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741855_1031 (size=12001) 2024-12-03T14:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235287956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:47,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/38b021f8f7384ace8dc33bfe4222953e 2024-12-03T14:13:47,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 is 50, key is test_row_0/B:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:48,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741856_1032 (size=12001) 2024-12-03T14:13:48,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 2024-12-03T14:13:48,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/77f55be78f1747c6ac1f918ee51bedf7 is 50, key is test_row_0/C:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741857_1033 (size=12001) 2024-12-03T14:13:48,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/77f55be78f1747c6ac1f918ee51bedf7 2024-12-03T14:13:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235288056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/38b021f8f7384ace8dc33bfe4222953e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e 2024-12-03T14:13:48,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235288066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235288066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235288066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T14:13:48,077 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:48,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:48,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T14:13:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T14:13:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T14:13:48,082 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 2024-12-03T14:13:48,084 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:48,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:48,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T14:13:48,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/77f55be78f1747c6ac1f918ee51bedf7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7 2024-12-03T14:13:48,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T14:13:48,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a9798ce308c36fb733235c6d0f2266b in 197ms, sequenceid=121, compaction requested=true 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:48,118 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:48,118 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:48,120 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:48,120 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:48,120 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:48,120 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:48,120 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:48,120 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:48,120 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/274ae10ba22e439a89629cad1ffa12ab, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=35.3 K 2024-12-03T14:13:48,120 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/4dee1ff4b11f4ac8a2f944086c937bd5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.6 K 2024-12-03T14:13:48,121 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 274ae10ba22e439a89629cad1ffa12ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:48,122 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dee1ff4b11f4ac8a2f944086c937bd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:48,122 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9b83dc0f774533b2b25955eb31c052, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733235226851 2024-12-03T14:13:48,123 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e5c0e631494caeb8f083c9a97692e3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733235226851 2024-12-03T14:13:48,123 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2c7a2d64de47a6b7e6fdfe7b3596e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:48,123 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b021f8f7384ace8dc33bfe4222953e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:48,142 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:48,142 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:48,143 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7189d84c1b2741b98cbdc938d2380ed1 is 50, key is test_row_0/B:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:48,143 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/f95c2ded326741709f9c8593e2ec4bff is 50, key is test_row_0/A:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:48,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741858_1034 (size=12241) 2024-12-03T14:13:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T14:13:48,187 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/f95c2ded326741709f9c8593e2ec4bff as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f95c2ded326741709f9c8593e2ec4bff 2024-12-03T14:13:48,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741859_1035 (size=12241) 2024-12-03T14:13:48,203 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7189d84c1b2741b98cbdc938d2380ed1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7189d84c1b2741b98cbdc938d2380ed1 2024-12-03T14:13:48,209 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into f95c2ded326741709f9c8593e2ec4bff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:48,209 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:48,209 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235228118; duration=0sec 2024-12-03T14:13:48,209 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:48,209 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:48,210 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:48,213 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:48,213 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:48,213 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:48,213 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b5d1da2f1f114d87b689d4ad486c5333, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=35.3 K 2024-12-03T14:13:48,214 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d1da2f1f114d87b689d4ad486c5333, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733235225630 2024-12-03T14:13:48,215 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8b8e9b19f8462d8f975296a17d74fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733235226851 2024-12-03T14:13:48,216 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f55be78f1747c6ac1f918ee51bedf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:48,229 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 7189d84c1b2741b98cbdc938d2380ed1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:48,230 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:48,230 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235228118; duration=0sec 2024-12-03T14:13:48,230 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:48,230 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:48,238 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:48,239 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/f2f49a2862974bab9b219115576b0b38 is 50, key is test_row_0/C:col10/1733235227277/Put/seqid=0 2024-12-03T14:13:48,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T14:13:48,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:48,241 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:13:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:48,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:48,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:48,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/f518eda8f77f41e4b6c38c858e77ffba is 50, key is test_row_0/A:col10/1733235227950/Put/seqid=0 2024-12-03T14:13:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:48,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:48,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741860_1036 (size=12241) 2024-12-03T14:13:48,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741861_1037 (size=12151) 2024-12-03T14:13:48,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235288320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235288323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235228309, max timestamp=1733235228327Num uniques 859; 2024-12-03T14:13:48,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235228309, max timestamp=1733235228327Num uniques 859; 2024-12-03T14:13:48,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:48,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:48,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235228309, max timestamp=1733235228327Num uniques 859; 2024-12-03T14:13:48,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235288379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235288381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T14:13:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235288446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235288447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235288467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235288497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235288497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235288656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235288657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,691 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/f518eda8f77f41e4b6c38c858e77ffba 2024-12-03T14:13:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T14:13:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235288707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235288707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:48,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:48,710 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/f2f49a2862974bab9b219115576b0b38 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f2f49a2862974bab9b219115576b0b38 2024-12-03T14:13:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 is 50, key is test_row_0/B:col10/1733235227950/Put/seqid=0 2024-12-03T14:13:48,725 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into f2f49a2862974bab9b219115576b0b38(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:48,725 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:48,725 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235228118; duration=0sec 2024-12-03T14:13:48,725 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:48,725 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:48,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741862_1038 (size=12151) 2024-12-03T14:13:48,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 2024-12-03T14:13:48,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55f91be7aae641609eec9c27820a6e91 is 50, key is test_row_0/C:col10/1733235227950/Put/seqid=0 2024-12-03T14:13:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741863_1039 (size=12151) 2024-12-03T14:13:48,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55f91be7aae641609eec9c27820a6e91 2024-12-03T14:13:48,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/f518eda8f77f41e4b6c38c858e77ffba as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba 2024-12-03T14:13:48,802 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba, entries=150, sequenceid=135, filesize=11.9 K 2024-12-03T14:13:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 2024-12-03T14:13:48,814 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9, entries=150, sequenceid=135, filesize=11.9 K 2024-12-03T14:13:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55f91be7aae641609eec9c27820a6e91 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91 2024-12-03T14:13:48,830 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91, entries=150, sequenceid=135, filesize=11.9 K 2024-12-03T14:13:48,831 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 8a9798ce308c36fb733235c6d0f2266b in 590ms, sequenceid=135, compaction requested=false 2024-12-03T14:13:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T14:13:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T14:13:48,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T14:13:48,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 750 msec 2024-12-03T14:13:48,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 762 msec 2024-12-03T14:13:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:48,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T14:13:48,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:48,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:48,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/61344512d2164e19a437115a907ce637 is 50, key is test_row_0/A:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741864_1040 (size=14541) 2024-12-03T14:13:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235289015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235289017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235289017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235289017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235289136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235289136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T14:13:49,216 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:49,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T14:13:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:49,220 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:49,221 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:49,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235289346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235289347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T14:13:49,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:49,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/61344512d2164e19a437115a907ce637 2024-12-03T14:13:49,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f730a7cf1a0144bab78bf04c9a136ea2 is 50, key is test_row_0/B:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741865_1041 (size=12151) 2024-12-03T14:13:49,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f730a7cf1a0144bab78bf04c9a136ea2 2024-12-03T14:13:49,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/4819d4851a1344e6973c5fdb56d9dc96 is 50, key is test_row_0/C:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235289476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741866_1042 (size=12151) 2024-12-03T14:13:49,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/4819d4851a1344e6973c5fdb56d9dc96 2024-12-03T14:13:49,484 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/61344512d2164e19a437115a907ce637 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637 2024-12-03T14:13:49,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637, entries=200, sequenceid=172, filesize=14.2 K 2024-12-03T14:13:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/f730a7cf1a0144bab78bf04c9a136ea2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2 2024-12-03T14:13:49,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T14:13:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/4819d4851a1344e6973c5fdb56d9dc96 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96 2024-12-03T14:13:49,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T14:13:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T14:13:49,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 8a9798ce308c36fb733235c6d0f2266b in 557ms, sequenceid=172, compaction requested=true 2024-12-03T14:13:49,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:49,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:49,533 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:49,533 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:49,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:49,537 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:49,537 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:49,537 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,537 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7189d84c1b2741b98cbdc938d2380ed1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=35.7 K 2024-12-03T14:13:49,537 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:49,538 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:49,538 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,538 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f95c2ded326741709f9c8593e2ec4bff, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=38.0 K 2024-12-03T14:13:49,538 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95c2ded326741709f9c8593e2ec4bff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:49,539 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f518eda8f77f41e4b6c38c858e77ffba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733235227935 2024-12-03T14:13:49,539 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61344512d2164e19a437115a907ce637, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228377 2024-12-03T14:13:49,540 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7189d84c1b2741b98cbdc938d2380ed1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:49,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,550 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0a1efb5a3e4e6a8265c8df500bcbc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733235227935 2024-12-03T14:13:49,551 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f730a7cf1a0144bab78bf04c9a136ea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228379 2024-12-03T14:13:49,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/daac6091f6cd4beba280587cd86db4db is 50, key is test_row_0/A:col10/1733235229012/Put/seqid=0 2024-12-03T14:13:49,582 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#31 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:49,587 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/a8149cf50e774593879b0f7a11509ba3 is 50, key is test_row_0/A:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,598 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#32 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:49,598 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/62843610b0664b09b004b8caf4a6749e is 50, key is test_row_0/B:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741867_1043 (size=12151) 2024-12-03T14:13:49,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/daac6091f6cd4beba280587cd86db4db 2024-12-03T14:13:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235289608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235289610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741869_1045 (size=12493) 2024-12-03T14:13:49,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741868_1044 (size=12493) 2024-12-03T14:13:49,647 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/62843610b0664b09b004b8caf4a6749e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/62843610b0664b09b004b8caf4a6749e 2024-12-03T14:13:49,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7ed9023699b046d78d780d794dd483e7 is 50, key is test_row_0/B:col10/1733235229012/Put/seqid=0 2024-12-03T14:13:49,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235289657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235289657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,664 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 62843610b0664b09b004b8caf4a6749e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:49,664 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:49,664 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235229533; duration=0sec 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,666 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:49,667 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:49,667 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:49,667 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/a8149cf50e774593879b0f7a11509ba3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/a8149cf50e774593879b0f7a11509ba3 2024-12-03T14:13:49,668 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:49,669 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:49,669 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,669 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f2f49a2862974bab9b219115576b0b38, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=35.7 K 2024-12-03T14:13:49,670 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f49a2862974bab9b219115576b0b38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733235227277 2024-12-03T14:13:49,670 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f91be7aae641609eec9c27820a6e91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733235227935 2024-12-03T14:13:49,671 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4819d4851a1344e6973c5fdb56d9dc96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228379 2024-12-03T14:13:49,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741870_1046 (size=12151) 2024-12-03T14:13:49,681 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into a8149cf50e774593879b0f7a11509ba3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:49,681 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:49,681 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235229532; duration=0sec 2024-12-03T14:13:49,681 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:49,681 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:49,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T14:13:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:49,695 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:49,696 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/db7dd28928304509869b2910762176f9 is 50, key is test_row_0/C:col10/1733235228379/Put/seqid=0 2024-12-03T14:13:49,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7ed9023699b046d78d780d794dd483e7 2024-12-03T14:13:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741871_1047 (size=12493) 2024-12-03T14:13:49,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/155383e81f9545a99dde5246b91f21bf is 50, key is test_row_0/C:col10/1733235229012/Put/seqid=0 2024-12-03T14:13:49,725 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/db7dd28928304509869b2910762176f9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/db7dd28928304509869b2910762176f9 2024-12-03T14:13:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235289726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741872_1048 (size=12151) 2024-12-03T14:13:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235289727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/155383e81f9545a99dde5246b91f21bf 2024-12-03T14:13:49,737 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into db7dd28928304509869b2910762176f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:49,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:49,737 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235229533; duration=0sec 2024-12-03T14:13:49,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:49,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:49,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/daac6091f6cd4beba280587cd86db4db as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db 2024-12-03T14:13:49,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T14:13:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7ed9023699b046d78d780d794dd483e7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7 2024-12-03T14:13:49,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T14:13:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/155383e81f9545a99dde5246b91f21bf as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf 2024-12-03T14:13:49,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T14:13:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a9798ce308c36fb733235c6d0f2266b in 237ms, sequenceid=185, compaction requested=false 2024-12-03T14:13:49,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:49,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T14:13:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:49,838 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:49,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/58d8cd8deb0644a8be9dfb3d37a57f23 is 50, key is test_row_0/A:col10/1733235229590/Put/seqid=0 2024-12-03T14:13:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:49,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741873_1049 (size=12151) 2024-12-03T14:13:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:49,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235289957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235289964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235290066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235290076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235290166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235290176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,275 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/58d8cd8deb0644a8be9dfb3d37a57f23 2024-12-03T14:13:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235290277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235290286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/120fc906ed6149a4a336e8e42c67f59e is 50, key is test_row_0/B:col10/1733235229590/Put/seqid=0 2024-12-03T14:13:50,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741874_1050 (size=12151) 2024-12-03T14:13:50,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235290586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235290606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:50,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/120fc906ed6149a4a336e8e42c67f59e 2024-12-03T14:13:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/79fd42a0214b44e58736a526172e8f99 is 50, key is test_row_0/C:col10/1733235229590/Put/seqid=0 2024-12-03T14:13:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741875_1051 (size=12151) 2024-12-03T14:13:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235291096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235291116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/79fd42a0214b44e58736a526172e8f99 2024-12-03T14:13:51,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/58d8cd8deb0644a8be9dfb3d37a57f23 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23 2024-12-03T14:13:51,154 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:13:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/120fc906ed6149a4a336e8e42c67f59e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e 2024-12-03T14:13:51,162 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:13:51,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/79fd42a0214b44e58736a526172e8f99 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99 2024-12-03T14:13:51,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:13:51,175 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8a9798ce308c36fb733235c6d0f2266b in 1337ms, sequenceid=207, compaction requested=true 2024-12-03T14:13:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T14:13:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T14:13:51,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T14:13:51,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-12-03T14:13:51,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9630 sec 2024-12-03T14:13:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:51,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:51,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ef76b4efeca04abaab5172d5f05a063b is 50, key is test_row_0/A:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741876_1052 (size=14541) 2024-12-03T14:13:51,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235291217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235291223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235291326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235291336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T14:13:51,366 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:51,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T14:13:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:51,369 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:51,370 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:51,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:51,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235291507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,511 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:51,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T14:13:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235291536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235291546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ef76b4efeca04abaab5172d5f05a063b 2024-12-03T14:13:51,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/438168cb1c9f42acb1ed1051404c27ae is 50, key is test_row_0/B:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:51,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741877_1053 (size=12151) 2024-12-03T14:13:51,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/438168cb1c9f42acb1ed1051404c27ae 2024-12-03T14:13:51,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a97222b7a1fe4146aaccb79a2cc29f5d is 50, key is test_row_0/C:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:51,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741878_1054 (size=12151) 2024-12-03T14:13:51,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a97222b7a1fe4146aaccb79a2cc29f5d 2024-12-03T14:13:51,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T14:13:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:51,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ef76b4efeca04abaab5172d5f05a063b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b 2024-12-03T14:13:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:51,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b, entries=200, sequenceid=222, filesize=14.2 K 2024-12-03T14:13:51,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/438168cb1c9f42acb1ed1051404c27ae as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae 2024-12-03T14:13:51,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae, entries=150, sequenceid=222, filesize=11.9 K 2024-12-03T14:13:51,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a97222b7a1fe4146aaccb79a2cc29f5d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d 2024-12-03T14:13:51,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d, entries=150, sequenceid=222, filesize=11.9 K 2024-12-03T14:13:51,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 8a9798ce308c36fb733235c6d0f2266b in 534ms, sequenceid=222, compaction requested=true 2024-12-03T14:13:51,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:51,723 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:51,723 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:51,724 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:51,725 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:51,725 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,725 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/a8149cf50e774593879b0f7a11509ba3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=50.1 K 2024-12-03T14:13:51,725 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:51,725 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:51,725 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,726 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/62843610b0664b09b004b8caf4a6749e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=47.8 K 2024-12-03T14:13:51,727 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8149cf50e774593879b0f7a11509ba3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228379 2024-12-03T14:13:51,727 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 62843610b0664b09b004b8caf4a6749e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228379 2024-12-03T14:13:51,728 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting daac6091f6cd4beba280587cd86db4db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733235229012 2024-12-03T14:13:51,728 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed9023699b046d78d780d794dd483e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733235229012 2024-12-03T14:13:51,729 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d8cd8deb0644a8be9dfb3d37a57f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235229590 2024-12-03T14:13:51,729 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 120fc906ed6149a4a336e8e42c67f59e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235229590 2024-12-03T14:13:51,729 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef76b4efeca04abaab5172d5f05a063b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:51,729 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 438168cb1c9f42acb1ed1051404c27ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:51,755 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:51,756 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/d32f21998ff44caa9aae0ddf39a094ae is 50, key is test_row_0/A:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:51,763 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:51,764 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e9f52cd92de147a38951678b854fdd3a is 50, key is test_row_0/B:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:51,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741879_1055 (size=12629) 2024-12-03T14:13:51,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741880_1056 (size=12629) 2024-12-03T14:13:51,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T14:13:51,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:51,833 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:51,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7342955538ee42d686cf42475213cd35 is 50, key is test_row_0/A:col10/1733235231219/Put/seqid=0 2024-12-03T14:13:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:51,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741881_1057 (size=12151) 2024-12-03T14:13:51,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235291896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235291896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:51,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:51,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235292006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235292006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235292106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,108 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2144 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235292126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,129 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,184 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/d32f21998ff44caa9aae0ddf39a094ae as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/d32f21998ff44caa9aae0ddf39a094ae 2024-12-03T14:13:52,193 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into d32f21998ff44caa9aae0ddf39a094ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:52,193 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:52,193 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=12, startTime=1733235231722; duration=0sec 2024-12-03T14:13:52,193 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:52,193 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:52,193 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:52,196 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:52,196 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:52,196 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:52,196 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/db7dd28928304509869b2910762176f9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=47.8 K 2024-12-03T14:13:52,197 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7dd28928304509869b2910762176f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235228379 2024-12-03T14:13:52,197 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155383e81f9545a99dde5246b91f21bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733235229012 2024-12-03T14:13:52,197 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79fd42a0214b44e58736a526172e8f99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235229590 2024-12-03T14:13:52,198 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97222b7a1fe4146aaccb79a2cc29f5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:52,202 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e9f52cd92de147a38951678b854fdd3a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e9f52cd92de147a38951678b854fdd3a 2024-12-03T14:13:52,210 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into e9f52cd92de147a38951678b854fdd3a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:52,211 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:52,211 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=12, startTime=1733235231723; duration=0sec 2024-12-03T14:13:52,211 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:52,211 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:52,216 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:52,217 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/0f3298a063714c61ba0e6775ea67b6b6 is 50, key is test_row_0/C:col10/1733235231188/Put/seqid=0 2024-12-03T14:13:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235292217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235292217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741882_1058 (size=12629) 2024-12-03T14:13:52,228 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/0f3298a063714c61ba0e6775ea67b6b6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/0f3298a063714c61ba0e6775ea67b6b6 2024-12-03T14:13:52,236 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into 0f3298a063714c61ba0e6775ea67b6b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:52,236 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:52,236 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=12, startTime=1733235231723; duration=0sec 2024-12-03T14:13:52,236 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:52,237 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:52,283 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7342955538ee42d686cf42475213cd35 2024-12-03T14:13:52,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/a9b8c52fe1d94c79931880c962c6b8b0 is 50, key is test_row_0/B:col10/1733235231219/Put/seqid=0 2024-12-03T14:13:52,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741883_1059 (size=12151) 2024-12-03T14:13:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235292526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235292526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:52,699 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/a9b8c52fe1d94c79931880c962c6b8b0 2024-12-03T14:13:52,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/721ff4ee22df44658c2932808be01d14 is 50, key is test_row_0/C:col10/1733235231219/Put/seqid=0 2024-12-03T14:13:52,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741884_1060 (size=12151) 2024-12-03T14:13:52,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/721ff4ee22df44658c2932808be01d14 2024-12-03T14:13:52,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7342955538ee42d686cf42475213cd35 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35 2024-12-03T14:13:52,727 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T14:13:52,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/a9b8c52fe1d94c79931880c962c6b8b0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0 2024-12-03T14:13:52,736 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T14:13:52,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/721ff4ee22df44658c2932808be01d14 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14 2024-12-03T14:13:52,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T14:13:52,747 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8a9798ce308c36fb733235c6d0f2266b in 914ms, sequenceid=239, compaction requested=false 2024-12-03T14:13:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T14:13:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T14:13:52,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T14:13:52,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3790 sec 2024-12-03T14:13:52,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3860 sec 2024-12-03T14:13:53,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:53,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:13:53,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:53,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ce4978816cc140f59863d26da11e6256 is 50, key is test_row_0/A:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741885_1061 (size=12251) 2024-12-03T14:13:53,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ce4978816cc140f59863d26da11e6256 2024-12-03T14:13:53,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/197576848ded40779873f69f37c183d8 is 50, key is test_row_0/B:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235293080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235293085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741886_1062 (size=12251) 2024-12-03T14:13:53,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/197576848ded40779873f69f37c183d8 2024-12-03T14:13:53,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3aa78313ae3e49f5a6950d6fca717468 is 50, key is test_row_0/C:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741887_1063 (size=12251) 2024-12-03T14:13:53,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3aa78313ae3e49f5a6950d6fca717468 2024-12-03T14:13:53,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/ce4978816cc140f59863d26da11e6256 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256 2024-12-03T14:13:53,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256, entries=150, sequenceid=259, filesize=12.0 K 2024-12-03T14:13:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/197576848ded40779873f69f37c183d8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8 2024-12-03T14:13:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8, entries=150, sequenceid=259, filesize=12.0 K 2024-12-03T14:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3aa78313ae3e49f5a6950d6fca717468 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468 2024-12-03T14:13:53,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468, entries=150, sequenceid=259, filesize=12.0 K 2024-12-03T14:13:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8a9798ce308c36fb733235c6d0f2266b in 147ms, sequenceid=259, compaction requested=true 2024-12-03T14:13:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:53,187 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:53,187 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:53,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:53,189 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:53,189 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:53,189 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:53,189 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:53,189 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,189 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,190 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/d32f21998ff44caa9aae0ddf39a094ae, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=36.2 K 2024-12-03T14:13:53,190 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e9f52cd92de147a38951678b854fdd3a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=36.2 K 2024-12-03T14:13:53,190 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f52cd92de147a38951678b854fdd3a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:53,191 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32f21998ff44caa9aae0ddf39a094ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:53,192 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7342955538ee42d686cf42475213cd35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235231214 2024-12-03T14:13:53,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b8c52fe1d94c79931880c962c6b8b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235231214 2024-12-03T14:13:53,193 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce4978816cc140f59863d26da11e6256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:53,193 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 197576848ded40779873f69f37c183d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:13:53,205 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:53,205 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/2d74902fc69a4f8994e74c464642ddc6 is 50, key is test_row_0/B:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:53,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:53,210 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:53,211 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8853736a1773402fa57b9e3ceb419cf0 is 50, key is test_row_0/A:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/49e41faabf8a4e49b31425e2731fc390 is 50, key is test_row_0/A:col10/1733235233201/Put/seqid=0 2024-12-03T14:13:53,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235293239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741888_1064 (size=12831) 2024-12-03T14:13:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235293245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741889_1065 (size=12831) 2024-12-03T14:13:53,284 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8853736a1773402fa57b9e3ceb419cf0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8853736a1773402fa57b9e3ceb419cf0 2024-12-03T14:13:53,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741890_1066 (size=14741) 2024-12-03T14:13:53,305 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into 8853736a1773402fa57b9e3ceb419cf0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:53,305 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:53,306 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235233186; duration=0sec 2024-12-03T14:13:53,306 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:53,306 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:53,306 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:53,308 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:53,308 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:53,308 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,308 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/0f3298a063714c61ba0e6775ea67b6b6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=36.2 K 2024-12-03T14:13:53,309 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f3298a063714c61ba0e6775ea67b6b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733235229953 2024-12-03T14:13:53,310 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721ff4ee22df44658c2932808be01d14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235231214 2024-12-03T14:13:53,312 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa78313ae3e49f5a6950d6fca717468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:53,344 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:53,345 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/ce9485eeef884915b2e97b222ea68fc1 is 50, key is test_row_0/C:col10/1733235231877/Put/seqid=0 2024-12-03T14:13:53,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235293358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235293367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741891_1067 (size=12831) 2024-12-03T14:13:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T14:13:53,517 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:53,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T14:13:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:53,520 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:53,521 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:53,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235293566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235293576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:53,667 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/2d74902fc69a4f8994e74c464642ddc6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2d74902fc69a4f8994e74c464642ddc6 2024-12-03T14:13:53,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:53,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:53,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,675 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 2d74902fc69a4f8994e74c464642ddc6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:53,675 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:53,675 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235233187; duration=0sec 2024-12-03T14:13:53,675 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:53,675 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/49e41faabf8a4e49b31425e2731fc390 2024-12-03T14:13:53,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7f550381830d45e7b07a90e3d5f7c76d is 50, key is test_row_0/B:col10/1733235233201/Put/seqid=0 2024-12-03T14:13:53,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741892_1068 (size=12301) 2024-12-03T14:13:53,793 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/ce9485eeef884915b2e97b222ea68fc1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/ce9485eeef884915b2e97b222ea68fc1 2024-12-03T14:13:53,800 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into ce9485eeef884915b2e97b222ea68fc1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:53,800 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:53,800 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235233187; duration=0sec 2024-12-03T14:13:53,801 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:53,801 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:53,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:53,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235293876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235293886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:53,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:53,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7f550381830d45e7b07a90e3d5f7c76d 2024-12-03T14:13:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235294126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:54,129 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:54,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/573503d1e3514dbcae41d32bc119a897 is 50, key is test_row_0/C:col10/1733235233201/Put/seqid=0 2024-12-03T14:13:54,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:54,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:54,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741893_1069 (size=12301) 2024-12-03T14:13:54,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235294156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:54,161 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:54,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235294386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235294396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:54,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:54,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/573503d1e3514dbcae41d32bc119a897 2024-12-03T14:13:54,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/49e41faabf8a4e49b31425e2731fc390 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390 2024-12-03T14:13:54,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390, entries=200, sequenceid=276, filesize=14.4 K 2024-12-03T14:13:54,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7f550381830d45e7b07a90e3d5f7c76d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d 2024-12-03T14:13:54,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T14:13:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/573503d1e3514dbcae41d32bc119a897 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897 2024-12-03T14:13:54,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T14:13:54,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8a9798ce308c36fb733235c6d0f2266b in 1397ms, sequenceid=276, compaction requested=false 2024-12-03T14:13:54,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:54,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T14:13:54,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:54,606 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:54,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:54,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/08ab124249c54c1abc28c0e2bb1b55a7 is 50, key is test_row_0/A:col10/1733235233237/Put/seqid=0 2024-12-03T14:13:54,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741894_1070 (size=12301) 2024-12-03T14:13:54,618 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/08ab124249c54c1abc28c0e2bb1b55a7 2024-12-03T14:13:54,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e3279fe9ab52466883151d9cbce46937 is 50, key is test_row_0/B:col10/1733235233237/Put/seqid=0 2024-12-03T14:13:54,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741895_1071 (size=12301) 2024-12-03T14:13:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:55,036 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e3279fe9ab52466883151d9cbce46937 2024-12-03T14:13:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/03e3f6a26e864e25b193945773f76fdc is 50, key is test_row_0/C:col10/1733235233237/Put/seqid=0 2024-12-03T14:13:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741896_1072 (size=12301) 2024-12-03T14:13:55,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/03e3f6a26e864e25b193945773f76fdc 2024-12-03T14:13:55,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/08ab124249c54c1abc28c0e2bb1b55a7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7 2024-12-03T14:13:55,069 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:13:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e3279fe9ab52466883151d9cbce46937 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937 2024-12-03T14:13:55,081 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:13:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/03e3f6a26e864e25b193945773f76fdc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc 2024-12-03T14:13:55,117 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:13:55,118 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 8a9798ce308c36fb733235c6d0f2266b in 512ms, sequenceid=294, compaction requested=true 2024-12-03T14:13:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T14:13:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T14:13:55,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T14:13:55,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-12-03T14:13:55,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6090 sec 2024-12-03T14:13:55,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:55,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/745719ddf28145cfa8733cf204893178 is 50, key is test_row_0/A:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:55,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741897_1073 (size=14737) 2024-12-03T14:13:55,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235235471, max timestamp=1733235235484Num uniques 859; 2024-12-03T14:13:55,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235235471, max timestamp=1733235235484Num uniques 859; 2024-12-03T14:13:55,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:13:55,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:13:55,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235235471, max timestamp=1733235235484Num uniques 859; 2024-12-03T14:13:55,496 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:13:55,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235295558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235295561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235295564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T14:13:55,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:55,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T14:13:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T14:13:55,670 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:55,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:55,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235295677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235295677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235295678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T14:13:55,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T14:13:55,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:55,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/745719ddf28145cfa8733cf204893178 2024-12-03T14:13:55,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4cfec4fe7f48436898d822796a1865f4 is 50, key is test_row_0/B:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:55,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741898_1074 (size=9857) 2024-12-03T14:13:55,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4cfec4fe7f48436898d822796a1865f4 2024-12-03T14:13:55,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235295886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235295886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235295886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:55,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9f8522afa52b4997b25619bfac8cfcb9 is 50, key is test_row_0/C:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:55,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741899_1075 (size=9857) 2024-12-03T14:13:55,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9f8522afa52b4997b25619bfac8cfcb9 2024-12-03T14:13:55,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/745719ddf28145cfa8733cf204893178 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178 2024-12-03T14:13:55,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178, entries=200, sequenceid=304, filesize=14.4 K 2024-12-03T14:13:55,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/4cfec4fe7f48436898d822796a1865f4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4 2024-12-03T14:13:55,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4, entries=100, sequenceid=304, filesize=9.6 K 2024-12-03T14:13:55,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/9f8522afa52b4997b25619bfac8cfcb9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9 2024-12-03T14:13:55,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9, entries=100, sequenceid=304, filesize=9.6 K 2024-12-03T14:13:55,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a9798ce308c36fb733235c6d0f2266b in 554ms, sequenceid=304, compaction requested=true 2024-12-03T14:13:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:55,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:55,976 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:55,976 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:55,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:55,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T14:13:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,981 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:55,981 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:55,981 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T14:13:55,981 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,982 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2d74902fc69a4f8994e74c464642ddc6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=46.2 K 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:55,982 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:55,982 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:55,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:55,982 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:55,982 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8853736a1773402fa57b9e3ceb419cf0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=53.3 K 2024-12-03T14:13:55,983 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d74902fc69a4f8994e74c464642ddc6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:55,983 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8853736a1773402fa57b9e3ceb419cf0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:55,984 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f550381830d45e7b07a90e3d5f7c76d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733235233073 2024-12-03T14:13:55,984 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e3279fe9ab52466883151d9cbce46937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235233230 2024-12-03T14:13:55,985 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e41faabf8a4e49b31425e2731fc390, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733235233052 2024-12-03T14:13:55,985 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cfec4fe7f48436898d822796a1865f4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235235411 2024-12-03T14:13:55,986 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ab124249c54c1abc28c0e2bb1b55a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235233230 2024-12-03T14:13:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T14:13:55,988 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745719ddf28145cfa8733cf204893178, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235235411 2024-12-03T14:13:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/936cdcc182534a4f938ee781249708ac is 50, key is test_row_0/A:col10/1733235235536/Put/seqid=0 2024-12-03T14:13:56,013 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,013 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,013 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5fb2c0dbec7e4d2e943df4235c2aeba7 is 50, key is test_row_0/A:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:56,013 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/203a0e36e5d2435bbe23a24a19831963 is 50, key is test_row_0/B:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:56,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741900_1076 (size=12301) 2024-12-03T14:13:56,017 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/936cdcc182534a4f938ee781249708ac 2024-12-03T14:13:56,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741901_1077 (size=13017) 2024-12-03T14:13:56,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cd1003ea070d4571a4397d902f9fc7ed is 50, key is test_row_0/B:col10/1733235235536/Put/seqid=0 2024-12-03T14:13:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741902_1078 (size=13017) 2024-12-03T14:13:56,079 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5fb2c0dbec7e4d2e943df4235c2aeba7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5fb2c0dbec7e4d2e943df4235c2aeba7 2024-12-03T14:13:56,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741903_1079 (size=12301) 2024-12-03T14:13:56,087 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into 5fb2c0dbec7e4d2e943df4235c2aeba7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,087 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cd1003ea070d4571a4397d902f9fc7ed 2024-12-03T14:13:56,088 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=12, startTime=1733235235976; duration=0sec 2024-12-03T14:13:56,088 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:56,088 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:56,088 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:56,090 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:56,090 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:56,090 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,090 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/ce9485eeef884915b2e97b222ea68fc1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=46.2 K 2024-12-03T14:13:56,090 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9485eeef884915b2e97b222ea68fc1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733235231877 2024-12-03T14:13:56,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573503d1e3514dbcae41d32bc119a897, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733235233073 2024-12-03T14:13:56,092 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e3f6a26e864e25b193945773f76fdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235233230 2024-12-03T14:13:56,094 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8522afa52b4997b25619bfac8cfcb9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235235411 2024-12-03T14:13:56,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/f077e32567464ac98c47869d70472384 is 50, key is test_row_0/C:col10/1733235235536/Put/seqid=0 2024-12-03T14:13:56,118 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,119 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a21c21adc525485f809baf218d6a999c is 50, key is test_row_0/C:col10/1733235235411/Put/seqid=0 2024-12-03T14:13:56,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741904_1080 (size=12301) 2024-12-03T14:13:56,126 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/f077e32567464ac98c47869d70472384 2024-12-03T14:13:56,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741905_1081 (size=13017) 2024-12-03T14:13:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/936cdcc182534a4f938ee781249708ac as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac 2024-12-03T14:13:56,143 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a21c21adc525485f809baf218d6a999c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a21c21adc525485f809baf218d6a999c 2024-12-03T14:13:56,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T14:13:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/cd1003ea070d4571a4397d902f9fc7ed as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed 2024-12-03T14:13:56,154 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into a21c21adc525485f809baf218d6a999c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,154 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,154 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=12, startTime=1733235235977; duration=0sec 2024-12-03T14:13:56,154 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:56,155 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:56,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T14:13:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/f077e32567464ac98c47869d70472384 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384 2024-12-03T14:13:56,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384, entries=150, sequenceid=341, filesize=12.0 K 2024-12-03T14:13:56,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 8a9798ce308c36fb733235c6d0f2266b in 186ms, sequenceid=341, compaction requested=false 2024-12-03T14:13:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T14:13:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T14:13:56,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T14:13:56,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 497 msec 2024-12-03T14:13:56,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 506 msec 2024-12-03T14:13:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:56,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:13:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/c1a83551bcd445bfa962d7a59fab2176 is 50, key is test_row_0/A:col10/1733235236207/Put/seqid=0 2024-12-03T14:13:56,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741906_1082 (size=17181) 2024-12-03T14:13:56,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/c1a83551bcd445bfa962d7a59fab2176 2024-12-03T14:13:56,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/0f03cdca73144373b02c30870e7a65ea is 50, key is test_row_0/B:col10/1733235236207/Put/seqid=0 2024-12-03T14:13:56,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235236228, max timestamp=1733235236248Num uniques 859; 2024-12-03T14:13:56,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:56,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235236228, max timestamp=1733235236248Num uniques 859; 2024-12-03T14:13:56,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235236228, max timestamp=1733235236248Num uniques 859; 2024-12-03T14:13:56,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:56,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:13:56,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741907_1083 (size=12301) 2024-12-03T14:13:56,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/0f03cdca73144373b02c30870e7a65ea 2024-12-03T14:13:56,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3636e93bdd7c44479e2972b5bb8df4de is 50, key is test_row_0/C:col10/1733235236207/Put/seqid=0 2024-12-03T14:13:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T14:13:56,296 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:56,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235296291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741908_1084 (size=12301) 2024-12-03T14:13:56,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3636e93bdd7c44479e2972b5bb8df4de 2024-12-03T14:13:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T14:13:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:56,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:56,303 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:56,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:56,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235296300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235296300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/c1a83551bcd445bfa962d7a59fab2176 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176 2024-12-03T14:13:56,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176, entries=250, sequenceid=354, filesize=16.8 K 2024-12-03T14:13:56,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/0f03cdca73144373b02c30870e7a65ea as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea 2024-12-03T14:13:56,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T14:13:56,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/3636e93bdd7c44479e2972b5bb8df4de as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de 2024-12-03T14:13:56,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T14:13:56,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 8a9798ce308c36fb733235c6d0f2266b in 129ms, sequenceid=354, compaction requested=true 2024-12-03T14:13:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:56,337 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:56,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:56,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:56,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T14:13:56,339 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:56,339 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:56,339 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,339 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5fb2c0dbec7e4d2e943df4235c2aeba7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=41.5 K 2024-12-03T14:13:56,340 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb2c0dbec7e4d2e943df4235c2aeba7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235233230 2024-12-03T14:13:56,340 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936cdcc182534a4f938ee781249708ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733235235536 2024-12-03T14:13:56,341 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a83551bcd445bfa962d7a59fab2176, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235236201 2024-12-03T14:13:56,359 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,360 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/9b9d493514e8465f9cac4c860f8f8bb6 is 50, key is test_row_0/A:col10/1733235236207/Put/seqid=0 2024-12-03T14:13:56,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741909_1085 (size=13119) 2024-12-03T14:13:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:56,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T14:13:56,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:56,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:56,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:56,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:56,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:56,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:56,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/abe43962226c4b44b41c5df6422966a9 is 50, key is test_row_0/A:col10/1733235236296/Put/seqid=0 2024-12-03T14:13:56,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741910_1086 (size=14741) 2024-12-03T14:13:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235296425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/abe43962226c4b44b41c5df6422966a9 2024-12-03T14:13:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235296431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235296431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/524ea27b99d948b2b3d923fbe19c0a36 is 50, key is test_row_0/B:col10/1733235236296/Put/seqid=0 2024-12-03T14:13:56,457 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/203a0e36e5d2435bbe23a24a19831963 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/203a0e36e5d2435bbe23a24a19831963 2024-12-03T14:13:56,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T14:13:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:56,468 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 203a0e36e5d2435bbe23a24a19831963(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,468 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,468 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=12, startTime=1733235235976; duration=0sec 2024-12-03T14:13:56,468 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T14:13:56,468 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:56,468 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:56,468 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741911_1087 (size=12301) 2024-12-03T14:13:56,471 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:56,471 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:56,471 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,471 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a21c21adc525485f809baf218d6a999c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=36.7 K 2024-12-03T14:13:56,472 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a21c21adc525485f809baf218d6a999c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235233230 2024-12-03T14:13:56,473 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f077e32567464ac98c47869d70472384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733235235536 2024-12-03T14:13:56,474 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3636e93bdd7c44479e2972b5bb8df4de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235236201 2024-12-03T14:13:56,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/524ea27b99d948b2b3d923fbe19c0a36 2024-12-03T14:13:56,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/b50b721d3ec54724bff07b4e315c21c5 is 50, key is test_row_0/C:col10/1733235236296/Put/seqid=0 2024-12-03T14:13:56,494 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,495 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7b7886aded7e47118d4dd14c0b88a894 is 50, key is test_row_0/C:col10/1733235236207/Put/seqid=0 2024-12-03T14:13:56,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741912_1088 (size=12301) 2024-12-03T14:13:56,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/b50b721d3ec54724bff07b4e315c21c5 2024-12-03T14:13:56,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741913_1089 (size=13119) 2024-12-03T14:13:56,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235296538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/abe43962226c4b44b41c5df6422966a9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9 2024-12-03T14:13:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235296546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235296546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9, entries=200, sequenceid=391, filesize=14.4 K 2024-12-03T14:13:56,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/524ea27b99d948b2b3d923fbe19c0a36 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36 2024-12-03T14:13:56,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36, entries=150, sequenceid=391, filesize=12.0 K 2024-12-03T14:13:56,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/b50b721d3ec54724bff07b4e315c21c5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5 2024-12-03T14:13:56,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5, entries=150, sequenceid=391, filesize=12.0 K 2024-12-03T14:13:56,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 8a9798ce308c36fb733235c6d0f2266b in 156ms, sequenceid=391, compaction requested=true 2024-12-03T14:13:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T14:13:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T14:13:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T14:13:56,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T14:13:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:13:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5539eddabd5443c3af51830a14ea9581 is 50, key is test_row_0/A:col10/1733235236429/Put/seqid=0 2024-12-03T14:13:56,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741914_1090 (size=12301) 2024-12-03T14:13:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:56,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:56,782 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/9b9d493514e8465f9cac4c860f8f8bb6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9b9d493514e8465f9cac4c860f8f8bb6 2024-12-03T14:13:56,792 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into 9b9d493514e8465f9cac4c860f8f8bb6(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,792 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,792 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235236337; duration=0sec 2024-12-03T14:13:56,792 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T14:13:56,792 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:56,792 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:56,792 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-03T14:13:56,794 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:13:56,794 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:13:56,794 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. because compaction request was cancelled 2024-12-03T14:13:56,794 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:56,794 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:56,798 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:56,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235236770, max timestamp=1733235236797Num uniques 859; 2024-12-03T14:13:56,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235236770, max timestamp=1733235236797Num uniques 859; 2024-12-03T14:13:56,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235236770, max timestamp=1733235236797Num uniques 859; 2024-12-03T14:13:56,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:56,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:56,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:56,800 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:56,800 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:56,801 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/203a0e36e5d2435bbe23a24a19831963, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=48.8 K 2024-12-03T14:13:56,801 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203a0e36e5d2435bbe23a24a19831963, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733235233230 2024-12-03T14:13:56,802 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd1003ea070d4571a4397d902f9fc7ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733235235536 2024-12-03T14:13:56,802 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f03cdca73144373b02c30870e7a65ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235236201 2024-12-03T14:13:56,802 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524ea27b99d948b2b3d923fbe19c0a36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733235236287 2024-12-03T14:13:56,814 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:56,815 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e8dd9e794c994e398e9840352ce5a07d is 50, key is test_row_0/B:col10/1733235236296/Put/seqid=0 2024-12-03T14:13:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741915_1091 (size=13153) 2024-12-03T14:13:56,832 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/e8dd9e794c994e398e9840352ce5a07d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e8dd9e794c994e398e9840352ce5a07d 2024-12-03T14:13:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235296831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235296832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,839 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into e8dd9e794c994e398e9840352ce5a07d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,840 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,840 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=12, startTime=1733235236564; duration=0sec 2024-12-03T14:13:56,840 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T14:13:56,840 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:56,840 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:56,840 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. because compaction request was cancelled 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. because compaction request was cancelled 2024-12-03T14:13:56,841 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:56,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235296836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:56,944 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7b7886aded7e47118d4dd14c0b88a894 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7b7886aded7e47118d4dd14c0b88a894 2024-12-03T14:13:56,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235296946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235296946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235296946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:56,954 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into 7b7886aded7e47118d4dd14c0b88a894(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:56,954 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:56,954 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235236338; duration=0sec 2024-12-03T14:13:56,954 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:56,954 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:57,022 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5539eddabd5443c3af51830a14ea9581 2024-12-03T14:13:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/024827e88c634e26993520f61c38a5f2 is 50, key is test_row_0/B:col10/1733235236429/Put/seqid=0 2024-12-03T14:13:57,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741916_1092 (size=12301) 2024-12-03T14:13:57,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235297156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235297156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235297156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:57,437 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/024827e88c634e26993520f61c38a5f2 2024-12-03T14:13:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 is 50, key is test_row_0/C:col10/1733235236429/Put/seqid=0 2024-12-03T14:13:57,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741917_1093 (size=12301) 2024-12-03T14:13:57,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235297466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235297466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235297466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 2024-12-03T14:13:57,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5539eddabd5443c3af51830a14ea9581 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581 2024-12-03T14:13:57,866 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T14:13:57,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/024827e88c634e26993520f61c38a5f2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2 2024-12-03T14:13:57,872 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T14:13:57,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 2024-12-03T14:13:57,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1, entries=150, sequenceid=402, filesize=12.0 K 2024-12-03T14:13:57,879 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8a9798ce308c36fb733235c6d0f2266b in 1266ms, sequenceid=402, compaction requested=true 2024-12-03T14:13:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T14:13:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T14:13:57,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T14:13:57,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-12-03T14:13:57,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5860 sec 2024-12-03T14:13:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:57,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:57,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:13:57,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6249856c63934c828cb17e49bb6b42df is 50, key is test_row_0/A:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:57,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741918_1094 (size=17181) 2024-12-03T14:13:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235297987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235297992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:57,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:57,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235297993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235298096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235298106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235298106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55446 deadline: 1733235298156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,159 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55462 deadline: 1733235298186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,190 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 8a9798ce308c36fb733235c6d0f2266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235298306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235298317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235298317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6249856c63934c828cb17e49bb6b42df 2024-12-03T14:13:58,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8109b7a68ff24103b9c3ee4e1c668c76 is 50, key is test_row_0/B:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:58,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741919_1095 (size=12301) 2024-12-03T14:13:58,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8109b7a68ff24103b9c3ee4e1c668c76 2024-12-03T14:13:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T14:13:58,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:13:58,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:13:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-03T14:13:58,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:13:58,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:13:58,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:13:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:13:58,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7d87cdb649194df0a2b01c71da248a23 is 50, key is test_row_0/C:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:58,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741920_1096 (size=12301) 2024-12-03T14:13:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:13:58,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T14:13:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235298617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235298636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235298636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:58,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T14:13:58,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:13:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:13:58,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7d87cdb649194df0a2b01c71da248a23 2024-12-03T14:13:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/6249856c63934c828cb17e49bb6b42df as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df 2024-12-03T14:13:58,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df, entries=250, sequenceid=442, filesize=16.8 K 2024-12-03T14:13:58,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8109b7a68ff24103b9c3ee4e1c668c76 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76 2024-12-03T14:13:58,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76, entries=150, sequenceid=442, filesize=12.0 K 2024-12-03T14:13:58,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/7d87cdb649194df0a2b01c71da248a23 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23 2024-12-03T14:13:58,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23, entries=150, sequenceid=442, filesize=12.0 K 2024-12-03T14:13:58,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 8a9798ce308c36fb733235c6d0f2266b in 918ms, sequenceid=442, compaction requested=true 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:58,898 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:58,898 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:13:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:58,899 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:13:58,899 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:13:58,899 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:58,899 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,899 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:13:58,899 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,899 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e8dd9e794c994e398e9840352ce5a07d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=36.9 K 2024-12-03T14:13:58,899 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9b9d493514e8465f9cac4c860f8f8bb6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=56.0 K 2024-12-03T14:13:58,900 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e8dd9e794c994e398e9840352ce5a07d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733235236287 2024-12-03T14:13:58,900 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9d493514e8465f9cac4c860f8f8bb6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235236201 2024-12-03T14:13:58,900 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 024827e88c634e26993520f61c38a5f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733235236411 2024-12-03T14:13:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe43962226c4b44b41c5df6422966a9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733235236287 2024-12-03T14:13:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8109b7a68ff24103b9c3ee4e1c668c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236830 2024-12-03T14:13:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5539eddabd5443c3af51830a14ea9581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733235236411 2024-12-03T14:13:58,903 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6249856c63934c828cb17e49bb6b42df, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236820 2024-12-03T14:13:58,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T14:13:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,916 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:13:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:13:58,922 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:58,923 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/455237bcbfed42eb8ef33110d3cb359b is 50, key is test_row_0/B:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:58,926 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:58,929 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/aa541f0a347441a4981c42915638a57b is 50, key is test_row_0/A:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/9d321a7162ce40bdba2ad0914f9e6251 is 50, key is test_row_0/A:col10/1733235237985/Put/seqid=0 2024-12-03T14:13:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741922_1098 (size=13255) 2024-12-03T14:13:58,967 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/aa541f0a347441a4981c42915638a57b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/aa541f0a347441a4981c42915638a57b 2024-12-03T14:13:58,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741923_1099 (size=12301) 2024-12-03T14:13:58,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741921_1097 (size=13255) 2024-12-03T14:13:58,983 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into aa541f0a347441a4981c42915638a57b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:58,984 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:58,984 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=12, startTime=1733235238898; duration=0sec 2024-12-03T14:13:58,984 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:13:58,984 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:13:58,984 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:13:58,986 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:13:58,986 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:13:58,986 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:13:58,986 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7b7886aded7e47118d4dd14c0b88a894, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=48.8 K 2024-12-03T14:13:58,987 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b7886aded7e47118d4dd14c0b88a894, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235236201 2024-12-03T14:13:58,988 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50b721d3ec54724bff07b4e315c21c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1733235236287 2024-12-03T14:13:58,989 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e5ca759e9a47f3a2b97fa115d3ddd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733235236411 2024-12-03T14:13:58,989 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d87cdb649194df0a2b01c71da248a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236830 2024-12-03T14:13:59,009 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:13:59,010 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/27b46d7522244a28890e51138ce0c13b is 50, key is test_row_0/C:col10/1733235236835/Put/seqid=0 2024-12-03T14:13:59,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741924_1100 (size=13255) 2024-12-03T14:13:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:13:59,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:13:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235239164, max timestamp=1733235239172Num uniques 859; 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235239164, max timestamp=1733235239172Num uniques 859; 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235239164, max timestamp=1733235239172Num uniques 859; 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:59,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:13:59,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235299200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235299200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235299201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235299306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235299306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235299306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/9d321a7162ce40bdba2ad0914f9e6251 2024-12-03T14:13:59,376 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/455237bcbfed42eb8ef33110d3cb359b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/455237bcbfed42eb8ef33110d3cb359b 2024-12-03T14:13:59,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/2fa6c043b84f4876bdb88a3a9c863c7a is 50, key is test_row_0/B:col10/1733235237985/Put/seqid=0 2024-12-03T14:13:59,382 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 455237bcbfed42eb8ef33110d3cb359b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:59,382 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:59,382 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235238898; duration=0sec 2024-12-03T14:13:59,382 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:59,382 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:13:59,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741925_1101 (size=12301) 2024-12-03T14:13:59,423 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/27b46d7522244a28890e51138ce0c13b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/27b46d7522244a28890e51138ce0c13b 2024-12-03T14:13:59,428 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into 27b46d7522244a28890e51138ce0c13b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:13:59,428 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:13:59,428 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=12, startTime=1733235238898; duration=0sec 2024-12-03T14:13:59,429 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:13:59,429 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:13:59,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235299516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235299516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235299516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:13:59,786 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/2fa6c043b84f4876bdb88a3a9c863c7a 2024-12-03T14:13:59,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/c420cac11c4a417bad7970c28399b17b is 50, key is test_row_0/C:col10/1733235237985/Put/seqid=0 2024-12-03T14:13:59,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741926_1102 (size=12301) 2024-12-03T14:13:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235299826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235299826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:13:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235299826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:13:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,200 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/c420cac11c4a417bad7970c28399b17b 2024-12-03T14:14:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/9d321a7162ce40bdba2ad0914f9e6251 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251 2024-12-03T14:14:00,209 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251, entries=150, sequenceid=451, filesize=12.0 K 2024-12-03T14:14:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/2fa6c043b84f4876bdb88a3a9c863c7a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a 2024-12-03T14:14:00,214 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a, entries=150, sequenceid=451, filesize=12.0 K 2024-12-03T14:14:00,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/c420cac11c4a417bad7970c28399b17b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b 2024-12-03T14:14:00,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b, entries=150, sequenceid=451, filesize=12.0 K 2024-12-03T14:14:00,221 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 8a9798ce308c36fb733235c6d0f2266b in 1306ms, sequenceid=451, compaction requested=false 2024-12-03T14:14:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-03T14:14:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-03T14:14:00,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T14:14:00,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-12-03T14:14:00,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7770 sec 2024-12-03T14:14:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:00,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5e5dd518d2c54873a5462ac727790b62 is 50, key is test_row_0/A:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235300347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235300349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235300349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741927_1103 (size=14741) 2024-12-03T14:14:00,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5e5dd518d2c54873a5462ac727790b62 2024-12-03T14:14:00,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/44355e6ce29d4f189125ebe3ffdae8cd is 50, key is test_row_0/B:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741928_1104 (size=12301) 2024-12-03T14:14:00,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/44355e6ce29d4f189125ebe3ffdae8cd 2024-12-03T14:14:00,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/d0340efb11e14d2ca1abbfc0df2b2b88 is 50, key is test_row_0/C:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741929_1105 (size=12301) 2024-12-03T14:14:00,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/d0340efb11e14d2ca1abbfc0df2b2b88 2024-12-03T14:14:00,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/5e5dd518d2c54873a5462ac727790b62 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62 2024-12-03T14:14:00,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62, entries=200, sequenceid=491, filesize=14.4 K 2024-12-03T14:14:00,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/44355e6ce29d4f189125ebe3ffdae8cd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd 2024-12-03T14:14:00,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd, entries=150, sequenceid=491, filesize=12.0 K 2024-12-03T14:14:00,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/d0340efb11e14d2ca1abbfc0df2b2b88 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88 2024-12-03T14:14:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88, entries=150, sequenceid=491, filesize=12.0 K 2024-12-03T14:14:00,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 8a9798ce308c36fb733235c6d0f2266b in 87ms, sequenceid=491, compaction requested=true 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:00,426 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:00,426 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:00,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:00,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:00,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:00,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:14:00,428 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,428 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/455237bcbfed42eb8ef33110d3cb359b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.0 K 2024-12-03T14:14:00,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 455237bcbfed42eb8ef33110d3cb359b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236830 2024-12-03T14:14:00,429 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:00,429 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:14:00,429 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,429 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/aa541f0a347441a4981c42915638a57b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=39.4 K 2024-12-03T14:14:00,429 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa6c043b84f4876bdb88a3a9c863c7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733235237985 2024-12-03T14:14:00,430 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa541f0a347441a4981c42915638a57b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236830 2024-12-03T14:14:00,430 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 44355e6ce29d4f189125ebe3ffdae8cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:00,430 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d321a7162ce40bdba2ad0914f9e6251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733235237985 2024-12-03T14:14:00,431 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e5dd518d2c54873a5462ac727790b62, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:00,457 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:00,458 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/e6d1f350e086404c9eca84872a739aff is 50, key is test_row_0/A:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:00,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:00,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:00,463 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:00,463 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/95d2056be1a842a4bbdde807ae8b1e20 is 50, key is test_row_0/B:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7b4ef583709248b09e85ef1831942a26 is 50, key is test_row_0/A:col10/1733235240456/Put/seqid=0 2024-12-03T14:14:00,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741930_1106 (size=13357) 2024-12-03T14:14:00,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235240486, max timestamp=1733235240514Num uniques 859; 2024-12-03T14:14:00,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235240486, max timestamp=1733235240514Num uniques 859; 2024-12-03T14:14:00,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:14:00,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:14:00,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235240486, max timestamp=1733235240514Num uniques 859; 2024-12-03T14:14:00,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:14:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741931_1107 (size=13357) 2024-12-03T14:14:00,557 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/95d2056be1a842a4bbdde807ae8b1e20 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/95d2056be1a842a4bbdde807ae8b1e20 2024-12-03T14:14:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741932_1108 (size=12301) 2024-12-03T14:14:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235300558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,568 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 95d2056be1a842a4bbdde807ae8b1e20(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:00,568 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,568 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235240426; duration=0sec 2024-12-03T14:14:00,569 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:00,569 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:14:00,569 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:00,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7b4ef583709248b09e85ef1831942a26 2024-12-03T14:14:00,571 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:00,571 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:14:00,571 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,572 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/27b46d7522244a28890e51138ce0c13b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.0 K 2024-12-03T14:14:00,573 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b46d7522244a28890e51138ce0c13b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235236830 2024-12-03T14:14:00,573 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c420cac11c4a417bad7970c28399b17b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733235237985 2024-12-03T14:14:00,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235300565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,574 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d0340efb11e14d2ca1abbfc0df2b2b88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235300568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/26af645637d0451d9205b48a292c0073 is 50, key is test_row_0/B:col10/1733235240456/Put/seqid=0 2024-12-03T14:14:00,602 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:00,603 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/30dc9b5bce2344188a530598cb84da5c is 50, key is test_row_0/C:col10/1733235239193/Put/seqid=0 2024-12-03T14:14:00,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741933_1109 (size=12301) 2024-12-03T14:14:00,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/26af645637d0451d9205b48a292c0073 2024-12-03T14:14:00,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741934_1110 (size=13357) 2024-12-03T14:14:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T14:14:00,616 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:00,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-03T14:14:00,621 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:00,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:00,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:00,626 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/30dc9b5bce2344188a530598cb84da5c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/30dc9b5bce2344188a530598cb84da5c 2024-12-03T14:14:00,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/606032d827a74617bb442d9d9479a899 is 50, key is test_row_0/C:col10/1733235240456/Put/seqid=0 2024-12-03T14:14:00,636 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into 30dc9b5bce2344188a530598cb84da5c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:00,636 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,636 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235240426; duration=0sec 2024-12-03T14:14:00,636 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:00,636 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:14:00,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741935_1111 (size=12301) 2024-12-03T14:14:00,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/606032d827a74617bb442d9d9479a899 2024-12-03T14:14:00,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/7b4ef583709248b09e85ef1831942a26 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26 2024-12-03T14:14:00,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T14:14:00,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/26af645637d0451d9205b48a292c0073 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073 2024-12-03T14:14:00,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T14:14:00,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/606032d827a74617bb442d9d9479a899 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899 2024-12-03T14:14:00,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899, entries=150, sequenceid=501, filesize=12.0 K 2024-12-03T14:14:00,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8a9798ce308c36fb733235c6d0f2266b in 219ms, sequenceid=501, compaction requested=false 2024-12-03T14:14:00,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:00,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-03T14:14:00,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:00,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:00,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:00,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:00,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/20ffc0d471354770bf9dc0f178b345a3 is 50, key is test_row_0/A:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741936_1112 (size=14741) 2024-12-03T14:14:00,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235300690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235300691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235300695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:00,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235300806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235300806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235300807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:00,911 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/e6d1f350e086404c9eca84872a739aff as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/e6d1f350e086404c9eca84872a739aff 2024-12-03T14:14:00,916 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into e6d1f350e086404c9eca84872a739aff(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:00,916 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:00,917 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235240426; duration=0sec 2024-12-03T14:14:00,917 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:00,917 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:14:00,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:00,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235301016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235301016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235301016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/20ffc0d471354770bf9dc0f178b345a3 2024-12-03T14:14:01,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/bd80900a48164de8b1ab0cdbc5f597cb is 50, key is test_row_0/B:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:01,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741937_1113 (size=12301) 2024-12-03T14:14:01,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235301326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235301326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235301326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/bd80900a48164de8b1ab0cdbc5f597cb 2024-12-03T14:14:01,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 is 50, key is test_row_0/C:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:01,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741938_1114 (size=12301) 2024-12-03T14:14:01,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:01,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235301836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235301836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235301836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:01,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:01,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:01,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 2024-12-03T14:14:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/20ffc0d471354770bf9dc0f178b345a3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3 2024-12-03T14:14:01,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3, entries=200, sequenceid=540, filesize=14.4 K 2024-12-03T14:14:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/bd80900a48164de8b1ab0cdbc5f597cb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb 2024-12-03T14:14:01,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb, entries=150, sequenceid=540, filesize=12.0 K 2024-12-03T14:14:01,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 2024-12-03T14:14:01,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8, entries=150, sequenceid=540, filesize=12.0 K 2024-12-03T14:14:01,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 8a9798ce308c36fb733235c6d0f2266b in 1275ms, sequenceid=540, compaction requested=true 2024-12-03T14:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:01,958 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:01,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:01,958 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:01,960 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:01,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:01,960 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:14:01,960 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,960 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:01,960 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/e6d1f350e086404c9eca84872a739aff, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=39.5 K 2024-12-03T14:14:01,960 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:14:01,960 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:01,960 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/95d2056be1a842a4bbdde807ae8b1e20, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.1 K 2024-12-03T14:14:01,960 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d1f350e086404c9eca84872a739aff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:01,961 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d2056be1a842a4bbdde807ae8b1e20, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:01,961 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4ef583709248b09e85ef1831942a26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733235240347 2024-12-03T14:14:01,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ffc0d471354770bf9dc0f178b345a3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:01,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 26af645637d0451d9205b48a292c0073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733235240347 2024-12-03T14:14:01,968 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting bd80900a48164de8b1ab0cdbc5f597cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:01,990 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#102 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:01,990 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/464fc9a9c7154185a7908090a8077cc2 is 50, key is test_row_0/A:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:01,993 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:01,994 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7121b468744946b4a64b62b35ad32b1f is 50, key is test_row_0/B:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:02,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T14:14:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:02,006 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T14:14:02,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:02,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741939_1115 (size=13459) 2024-12-03T14:14:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/75d07f734efa486cbd73694e4ec5ce54 is 50, key is test_row_0/A:col10/1733235240689/Put/seqid=0 2024-12-03T14:14:02,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741940_1116 (size=13459) 2024-12-03T14:14:02,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741941_1117 (size=12301) 2024-12-03T14:14:02,415 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7121b468744946b4a64b62b35ad32b1f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7121b468744946b4a64b62b35ad32b1f 2024-12-03T14:14:02,425 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 7121b468744946b4a64b62b35ad32b1f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:02,425 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:02,425 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235241958; duration=0sec 2024-12-03T14:14:02,425 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:02,425 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:14:02,425 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:02,427 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:02,427 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:14:02,427 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/75d07f734efa486cbd73694e4ec5ce54 2024-12-03T14:14:02,427 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:02,427 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/30dc9b5bce2344188a530598cb84da5c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.1 K 2024-12-03T14:14:02,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 30dc9b5bce2344188a530598cb84da5c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733235239193 2024-12-03T14:14:02,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 606032d827a74617bb442d9d9479a899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733235240347 2024-12-03T14:14:02,429 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e4fe4a6f9f3143fe97fe10cedadd1bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:02,433 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/464fc9a9c7154185a7908090a8077cc2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/464fc9a9c7154185a7908090a8077cc2 2024-12-03T14:14:02,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8f7a2021f2cc41f2b9bac8ca02fa2635 is 50, key is test_row_0/B:col10/1733235240689/Put/seqid=0 2024-12-03T14:14:02,442 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into 464fc9a9c7154185a7908090a8077cc2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:02,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:02,442 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235241958; duration=0sec 2024-12-03T14:14:02,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:02,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:14:02,450 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:02,452 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a1b0fd0247644360bae6f3ea02c89c64 is 50, key is test_row_0/C:col10/1733235240563/Put/seqid=0 2024-12-03T14:14:02,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741942_1118 (size=12301) 2024-12-03T14:14:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741943_1119 (size=13459) 2024-12-03T14:14:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:02,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. as already flushing 2024-12-03T14:14:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:02,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8f7a2021f2cc41f2b9bac8ca02fa2635 2024-12-03T14:14:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55029f98dc574de5ac6723ad526f89d1 is 50, key is test_row_0/C:col10/1733235240689/Put/seqid=0 2024-12-03T14:14:02,870 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/a1b0fd0247644360bae6f3ea02c89c64 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a1b0fd0247644360bae6f3ea02c89c64 2024-12-03T14:14:02,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741944_1120 (size=12301) 2024-12-03T14:14:02,875 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55029f98dc574de5ac6723ad526f89d1 2024-12-03T14:14:02,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235242869, max timestamp=1733235242877Num uniques 859; 2024-12-03T14:14:02,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235242869, max timestamp=1733235242877Num uniques 859; 2024-12-03T14:14:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235242869, max timestamp=1733235242877Num uniques 859; 2024-12-03T14:14:02,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:02,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/75d07f734efa486cbd73694e4ec5ce54 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54 2024-12-03T14:14:02,886 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into a1b0fd0247644360bae6f3ea02c89c64(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:02,886 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:02,886 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235241958; duration=0sec 2024-12-03T14:14:02,886 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:02,886 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:14:02,895 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T14:14:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/8f7a2021f2cc41f2b9bac8ca02fa2635 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635 2024-12-03T14:14:02,904 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T14:14:02,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/55029f98dc574de5ac6723ad526f89d1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1 2024-12-03T14:14:02,913 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T14:14:02,915 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=134.18 KB/137400 for 8a9798ce308c36fb733235c6d0f2266b in 908ms, sequenceid=549, compaction requested=false 2024-12-03T14:14:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-03T14:14:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-03T14:14:02,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-03T14:14:02,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:02,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:14:02,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:02,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:14:02,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:02,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:14:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:02,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T14:14:02,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2960 sec 2024-12-03T14:14:02,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3050 sec 2024-12-03T14:14:02,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8c6f53e961a74af5b67a95068372983e is 50, key is test_row_0/A:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235302927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:02,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235302927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:02,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235302929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:02,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:02,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741945_1121 (size=22065) 2024-12-03T14:14:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235303036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235303036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235303036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235303246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235303246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235303246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8c6f53e961a74af5b67a95068372983e 2024-12-03T14:14:03,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/ae15f4b2e3c74774a1c78784c379ea5b is 50, key is test_row_0/B:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:03,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741946_1122 (size=12301) 2024-12-03T14:14:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235303556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235303556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235303556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:03,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/ae15f4b2e3c74774a1c78784c379ea5b 2024-12-03T14:14:03,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4a806c26a5649e797df5d9c28226b68 is 50, key is test_row_0/C:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:03,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741947_1123 (size=12301) 2024-12-03T14:14:03,925 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T14:14:03,925 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:03,925 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T14:14:03,925 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:03,926 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T14:14:03,926 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:03,929 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,929 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,929 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,930 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T14:14:03,930 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,930 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:03,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:03,930 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:03,930 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:03,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:03,930 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:03,930 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:04,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55438 deadline: 1733235304066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:04,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55464 deadline: 1733235304067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55484 deadline: 1733235304067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:04,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:04,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4a806c26a5649e797df5d9c28226b68 2024-12-03T14:14:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8c6f53e961a74af5b67a95068372983e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e 2024-12-03T14:14:04,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e, entries=350, sequenceid=590, filesize=21.5 K 2024-12-03T14:14:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/ae15f4b2e3c74774a1c78784c379ea5b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b 2024-12-03T14:14:04,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b, entries=150, sequenceid=590, filesize=12.0 K 2024-12-03T14:14:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/e4a806c26a5649e797df5d9c28226b68 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68 2024-12-03T14:14:04,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68, entries=150, sequenceid=590, filesize=12.0 K 2024-12-03T14:14:04,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8a9798ce308c36fb733235c6d0f2266b in 1360ms, sequenceid=590, compaction requested=true 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:04,277 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:04,277 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a9798ce308c36fb733235c6d0f2266b:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:04,278 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:04,278 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47825 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:04,278 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/A is initiating minor compaction (all files) 2024-12-03T14:14:04,278 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/B is initiating minor compaction (all files) 2024-12-03T14:14:04,278 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/B in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:04,278 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/A in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:04,278 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7121b468744946b4a64b62b35ad32b1f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.2 K 2024-12-03T14:14:04,278 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/464fc9a9c7154185a7908090a8077cc2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=46.7 K 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464fc9a9c7154185a7908090a8077cc2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7121b468744946b4a64b62b35ad32b1f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75d07f734efa486cbd73694e4ec5ce54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733235240689 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7a2021f2cc41f2b9bac8ca02fa2635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733235240689 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c6f53e961a74af5b67a95068372983e, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733235242901 2024-12-03T14:14:04,279 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ae15f4b2e3c74774a1c78784c379ea5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733235242910 2024-12-03T14:14:04,300 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#B#compaction#111 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:04,301 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7d9163a77fac4f84a08ccf8b85adf9ea is 50, key is test_row_0/B:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:04,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741948_1124 (size=13561) 2024-12-03T14:14:04,315 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#A#compaction#112 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:04,315 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/bf004550a2504f73a1b5fda661767255 is 50, key is test_row_0/A:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:04,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741949_1125 (size=13561) 2024-12-03T14:14:04,714 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/7d9163a77fac4f84a08ccf8b85adf9ea as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7d9163a77fac4f84a08ccf8b85adf9ea 2024-12-03T14:14:04,721 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/B of 8a9798ce308c36fb733235c6d0f2266b into 7d9163a77fac4f84a08ccf8b85adf9ea(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:04,721 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:04,721 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/B, priority=13, startTime=1733235244277; duration=0sec 2024-12-03T14:14:04,721 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:04,721 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:B 2024-12-03T14:14:04,721 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:04,723 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:04,723 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 8a9798ce308c36fb733235c6d0f2266b/C is initiating minor compaction (all files) 2024-12-03T14:14:04,723 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a9798ce308c36fb733235c6d0f2266b/C in TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:04,723 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a1b0fd0247644360bae6f3ea02c89c64, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp, totalSize=37.2 K 2024-12-03T14:14:04,724 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b0fd0247644360bae6f3ea02c89c64, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1733235240546 2024-12-03T14:14:04,724 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 55029f98dc574de5ac6723ad526f89d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1733235240689 2024-12-03T14:14:04,725 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a806c26a5649e797df5d9c28226b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733235242910 2024-12-03T14:14:04,727 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/bf004550a2504f73a1b5fda661767255 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/bf004550a2504f73a1b5fda661767255 2024-12-03T14:14:04,733 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/A of 8a9798ce308c36fb733235c6d0f2266b into bf004550a2504f73a1b5fda661767255(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:04,733 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:04,733 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/A, priority=13, startTime=1733235244277; duration=0sec 2024-12-03T14:14:04,733 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:04,733 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:A 2024-12-03T14:14:04,737 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a9798ce308c36fb733235c6d0f2266b#C#compaction#113 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:04,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/1c87c5e6fd8347d2a0aaad8466a3c7d8 is 50, key is test_row_0/C:col10/1733235242911/Put/seqid=0 2024-12-03T14:14:04,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741950_1126 (size=13561) 2024-12-03T14:14:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T14:14:04,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:05,092 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T14:14:05,092 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:05,092 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T14:14:05,092 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,092 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:05,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,092 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:05,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:14:05,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:05,093 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:05,093 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:05,093 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:05,093 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:05,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:05,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:05,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3800ec35f9204fe9b867cdb3c6615ec5 is 50, key is test_row_0/A:col10/1733235245088/Put/seqid=0 2024-12-03T14:14:05,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741951_1127 (size=12301) 2024-12-03T14:14:05,149 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/1c87c5e6fd8347d2a0aaad8466a3c7d8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1c87c5e6fd8347d2a0aaad8466a3c7d8 2024-12-03T14:14:05,156 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a9798ce308c36fb733235c6d0f2266b/C of 8a9798ce308c36fb733235c6d0f2266b into 1c87c5e6fd8347d2a0aaad8466a3c7d8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:05,156 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:05,156 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., storeName=8a9798ce308c36fb733235c6d0f2266b/C, priority=13, startTime=1733235244277; duration=0sec 2024-12-03T14:14:05,157 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:05,157 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a9798ce308c36fb733235c6d0f2266b:C 2024-12-03T14:14:05,457 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:14:05,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3800ec35f9204fe9b867cdb3c6615ec5 2024-12-03T14:14:05,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/684f1482b88f403ea8cba4b8b8d83df8 is 50, key is test_row_0/B:col10/1733235245088/Put/seqid=0 2024-12-03T14:14:05,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741952_1128 (size=12301) 2024-12-03T14:14:05,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/684f1482b88f403ea8cba4b8b8d83df8 2024-12-03T14:14:05,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8a653feec2a647b4aa7fb6f939d107d8 is 50, key is test_row_0/C:col10/1733235245088/Put/seqid=0 2024-12-03T14:14:05,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741953_1129 (size=12301) 2024-12-03T14:14:06,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8a653feec2a647b4aa7fb6f939d107d8 2024-12-03T14:14:06,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/3800ec35f9204fe9b867cdb3c6615ec5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3800ec35f9204fe9b867cdb3c6615ec5 2024-12-03T14:14:06,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3800ec35f9204fe9b867cdb3c6615ec5, entries=150, sequenceid=604, filesize=12.0 K 2024-12-03T14:14:06,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/684f1482b88f403ea8cba4b8b8d83df8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/684f1482b88f403ea8cba4b8b8d83df8 2024-12-03T14:14:06,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/684f1482b88f403ea8cba4b8b8d83df8, entries=150, sequenceid=604, filesize=12.0 K 2024-12-03T14:14:06,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/8a653feec2a647b4aa7fb6f939d107d8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8a653feec2a647b4aa7fb6f939d107d8 2024-12-03T14:14:06,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8a653feec2a647b4aa7fb6f939d107d8, entries=150, sequenceid=604, filesize=12.0 K 2024-12-03T14:14:06,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 8a9798ce308c36fb733235c6d0f2266b in 1277ms, sequenceid=604, compaction requested=false 2024-12-03T14:14:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a9798ce308c36fb733235c6d0f2266b: 2024-12-03T14:14:08,197 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T14:14:08,197 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:08,197 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,197 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:08,207 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T14:14:08,207 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:08,208 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,208 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 145 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3700 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3775 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1724 2024-12-03T14:14:08,208 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5168 rows 2024-12-03T14:14:08,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1708 2024-12-03T14:14:08,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5117 rows 2024-12-03T14:14:08,209 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:14:08,209 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:14:08,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:08,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:08,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:14:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:08,222 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235248221"}]},"ts":"1733235248221"} 2024-12-03T14:14:08,223 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:14:08,223 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:14:08,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:14:08,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, UNASSIGN}] 2024-12-03T14:14:08,229 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, UNASSIGN 2024-12-03T14:14:08,231 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8a9798ce308c36fb733235c6d0f2266b, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:08,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, UNASSIGN because future has completed 2024-12-03T14:14:08,233 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:14:08,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:08,390 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:08,390 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:14:08,390 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 8a9798ce308c36fb733235c6d0f2266b, disabling compactions & flushes 2024-12-03T14:14:08,390 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:08,390 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:08,390 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. after waiting 0 ms 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:08,391 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 8a9798ce308c36fb733235c6d0f2266b 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=A 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=B 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a9798ce308c36fb733235c6d0f2266b, store=C 2024-12-03T14:14:08,391 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:08,396 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8d491c2f2848499c8902a60e991fa911 is 50, key is test_row_2/A:col10/1733235248206/Put/seqid=0 2024-12-03T14:14:08,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741954_1130 (size=7415) 2024-12-03T14:14:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:08,804 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8d491c2f2848499c8902a60e991fa911 2024-12-03T14:14:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:08,854 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/c260f59e06664ce09a3813e6943aabe1 is 50, key is test_row_2/B:col10/1733235248206/Put/seqid=0 2024-12-03T14:14:08,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741955_1131 (size=7415) 2024-12-03T14:14:09,259 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/c260f59e06664ce09a3813e6943aabe1 2024-12-03T14:14:09,267 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/13ef626c21d94b5bb40109da22a11791 is 50, key is test_row_2/C:col10/1733235248206/Put/seqid=0 2024-12-03T14:14:09,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741956_1132 (size=7415) 2024-12-03T14:14:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:09,678 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/13ef626c21d94b5bb40109da22a11791 2024-12-03T14:14:09,684 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/A/8d491c2f2848499c8902a60e991fa911 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8d491c2f2848499c8902a60e991fa911 2024-12-03T14:14:09,691 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8d491c2f2848499c8902a60e991fa911, entries=50, sequenceid=610, filesize=7.2 K 2024-12-03T14:14:09,692 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/B/c260f59e06664ce09a3813e6943aabe1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/c260f59e06664ce09a3813e6943aabe1 2024-12-03T14:14:09,698 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/c260f59e06664ce09a3813e6943aabe1, entries=50, sequenceid=610, filesize=7.2 K 2024-12-03T14:14:09,699 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/.tmp/C/13ef626c21d94b5bb40109da22a11791 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/13ef626c21d94b5bb40109da22a11791 2024-12-03T14:14:09,705 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/13ef626c21d94b5bb40109da22a11791, entries=50, sequenceid=610, filesize=7.2 K 2024-12-03T14:14:09,707 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8a9798ce308c36fb733235c6d0f2266b in 1315ms, sequenceid=610, compaction requested=true 2024-12-03T14:14:09,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/4dee1ff4b11f4ac8a2f944086c937bd5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f95c2ded326741709f9c8593e2ec4bff, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/a8149cf50e774593879b0f7a11509ba3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/d32f21998ff44caa9aae0ddf39a094ae, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8853736a1773402fa57b9e3ceb419cf0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5fb2c0dbec7e4d2e943df4235c2aeba7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9b9d493514e8465f9cac4c860f8f8bb6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/aa541f0a347441a4981c42915638a57b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/e6d1f350e086404c9eca84872a739aff, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/464fc9a9c7154185a7908090a8077cc2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e] to archive 2024-12-03T14:14:09,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:09,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/1d76f19db3ef4bdf9026df6673ac0ffa 2024-12-03T14:14:09,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6130d50907424b799d79f6831d553355 2024-12-03T14:14:09,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/45862e0707444414b5c985306d87f180 2024-12-03T14:14:09,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/4dee1ff4b11f4ac8a2f944086c937bd5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/4dee1ff4b11f4ac8a2f944086c937bd5 2024-12-03T14:14:09,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3de22003861746c2815a99a3186a6ab4 2024-12-03T14:14:09,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/03e5c0e631494caeb8f083c9a97692e3 2024-12-03T14:14:09,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f95c2ded326741709f9c8593e2ec4bff to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f95c2ded326741709f9c8593e2ec4bff 2024-12-03T14:14:09,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/38b021f8f7384ace8dc33bfe4222953e 2024-12-03T14:14:09,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/f518eda8f77f41e4b6c38c858e77ffba 2024-12-03T14:14:09,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/61344512d2164e19a437115a907ce637 2024-12-03T14:14:09,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/a8149cf50e774593879b0f7a11509ba3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/a8149cf50e774593879b0f7a11509ba3 2024-12-03T14:14:09,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/daac6091f6cd4beba280587cd86db4db 2024-12-03T14:14:09,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/58d8cd8deb0644a8be9dfb3d37a57f23 2024-12-03T14:14:09,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ef76b4efeca04abaab5172d5f05a063b 2024-12-03T14:14:09,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/d32f21998ff44caa9aae0ddf39a094ae to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/d32f21998ff44caa9aae0ddf39a094ae 2024-12-03T14:14:09,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7342955538ee42d686cf42475213cd35 2024-12-03T14:14:09,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8853736a1773402fa57b9e3ceb419cf0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8853736a1773402fa57b9e3ceb419cf0 2024-12-03T14:14:09,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/ce4978816cc140f59863d26da11e6256 2024-12-03T14:14:09,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/49e41faabf8a4e49b31425e2731fc390 2024-12-03T14:14:09,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/08ab124249c54c1abc28c0e2bb1b55a7 2024-12-03T14:14:09,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/745719ddf28145cfa8733cf204893178 2024-12-03T14:14:09,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5fb2c0dbec7e4d2e943df4235c2aeba7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5fb2c0dbec7e4d2e943df4235c2aeba7 2024-12-03T14:14:09,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/936cdcc182534a4f938ee781249708ac 2024-12-03T14:14:09,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/c1a83551bcd445bfa962d7a59fab2176 2024-12-03T14:14:09,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9b9d493514e8465f9cac4c860f8f8bb6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9b9d493514e8465f9cac4c860f8f8bb6 2024-12-03T14:14:09,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/abe43962226c4b44b41c5df6422966a9 2024-12-03T14:14:09,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5539eddabd5443c3af51830a14ea9581 2024-12-03T14:14:09,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/6249856c63934c828cb17e49bb6b42df 2024-12-03T14:14:09,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/aa541f0a347441a4981c42915638a57b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/aa541f0a347441a4981c42915638a57b 2024-12-03T14:14:09,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/9d321a7162ce40bdba2ad0914f9e6251 2024-12-03T14:14:09,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/5e5dd518d2c54873a5462ac727790b62 2024-12-03T14:14:09,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/e6d1f350e086404c9eca84872a739aff to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/e6d1f350e086404c9eca84872a739aff 2024-12-03T14:14:09,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/7b4ef583709248b09e85ef1831942a26 2024-12-03T14:14:09,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/20ffc0d471354770bf9dc0f178b345a3 2024-12-03T14:14:09,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/464fc9a9c7154185a7908090a8077cc2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/464fc9a9c7154185a7908090a8077cc2 2024-12-03T14:14:09,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/75d07f734efa486cbd73694e4ec5ce54 2024-12-03T14:14:09,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8c6f53e961a74af5b67a95068372983e 2024-12-03T14:14:09,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/274ae10ba22e439a89629cad1ffa12ab, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7189d84c1b2741b98cbdc938d2380ed1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/62843610b0664b09b004b8caf4a6749e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e9f52cd92de147a38951678b854fdd3a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2d74902fc69a4f8994e74c464642ddc6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/203a0e36e5d2435bbe23a24a19831963, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e8dd9e794c994e398e9840352ce5a07d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/455237bcbfed42eb8ef33110d3cb359b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/95d2056be1a842a4bbdde807ae8b1e20, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7121b468744946b4a64b62b35ad32b1f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b] to archive 2024-12-03T14:14:09,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:09,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/6c38c92714894ec09fdb6a68704931a2 2024-12-03T14:14:09,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/d8818edb230248a588d21173c86c67ce 2024-12-03T14:14:09,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f7e5c0c1c00d4e9fb5aad86809fb3345 2024-12-03T14:14:09,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/274ae10ba22e439a89629cad1ffa12ab to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/274ae10ba22e439a89629cad1ffa12ab 2024-12-03T14:14:09,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/31e6b577e3a143a5935c2594e1615809 2024-12-03T14:14:09,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cf9b83dc0f774533b2b25955eb31c052 2024-12-03T14:14:09,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7189d84c1b2741b98cbdc938d2380ed1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7189d84c1b2741b98cbdc938d2380ed1 2024-12-03T14:14:09,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/1d2c7a2d64de47a6b7e6fdfe7b3596e3 2024-12-03T14:14:09,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4a0a1efb5a3e4e6a8265c8df500bcbc9 2024-12-03T14:14:09,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/62843610b0664b09b004b8caf4a6749e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/62843610b0664b09b004b8caf4a6749e 2024-12-03T14:14:09,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/f730a7cf1a0144bab78bf04c9a136ea2 2024-12-03T14:14:09,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7ed9023699b046d78d780d794dd483e7 2024-12-03T14:14:09,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/120fc906ed6149a4a336e8e42c67f59e 2024-12-03T14:14:09,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e9f52cd92de147a38951678b854fdd3a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e9f52cd92de147a38951678b854fdd3a 2024-12-03T14:14:09,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/438168cb1c9f42acb1ed1051404c27ae 2024-12-03T14:14:09,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/a9b8c52fe1d94c79931880c962c6b8b0 2024-12-03T14:14:09,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2d74902fc69a4f8994e74c464642ddc6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2d74902fc69a4f8994e74c464642ddc6 2024-12-03T14:14:09,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/197576848ded40779873f69f37c183d8 2024-12-03T14:14:09,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7f550381830d45e7b07a90e3d5f7c76d 2024-12-03T14:14:09,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e3279fe9ab52466883151d9cbce46937 2024-12-03T14:14:09,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/203a0e36e5d2435bbe23a24a19831963 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/203a0e36e5d2435bbe23a24a19831963 2024-12-03T14:14:09,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/4cfec4fe7f48436898d822796a1865f4 2024-12-03T14:14:09,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/cd1003ea070d4571a4397d902f9fc7ed 2024-12-03T14:14:09,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/0f03cdca73144373b02c30870e7a65ea 2024-12-03T14:14:09,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e8dd9e794c994e398e9840352ce5a07d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/e8dd9e794c994e398e9840352ce5a07d 2024-12-03T14:14:09,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/524ea27b99d948b2b3d923fbe19c0a36 2024-12-03T14:14:09,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/024827e88c634e26993520f61c38a5f2 2024-12-03T14:14:09,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/455237bcbfed42eb8ef33110d3cb359b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/455237bcbfed42eb8ef33110d3cb359b 2024-12-03T14:14:09,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8109b7a68ff24103b9c3ee4e1c668c76 2024-12-03T14:14:09,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/2fa6c043b84f4876bdb88a3a9c863c7a 2024-12-03T14:14:09,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/95d2056be1a842a4bbdde807ae8b1e20 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/95d2056be1a842a4bbdde807ae8b1e20 2024-12-03T14:14:09,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/44355e6ce29d4f189125ebe3ffdae8cd 2024-12-03T14:14:09,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/26af645637d0451d9205b48a292c0073 2024-12-03T14:14:09,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7121b468744946b4a64b62b35ad32b1f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7121b468744946b4a64b62b35ad32b1f 2024-12-03T14:14:09,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/bd80900a48164de8b1ab0cdbc5f597cb 2024-12-03T14:14:09,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/8f7a2021f2cc41f2b9bac8ca02fa2635 2024-12-03T14:14:09,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/ae15f4b2e3c74774a1c78784c379ea5b 2024-12-03T14:14:09,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b5d1da2f1f114d87b689d4ad486c5333, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f2f49a2862974bab9b219115576b0b38, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/db7dd28928304509869b2910762176f9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/0f3298a063714c61ba0e6775ea67b6b6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/ce9485eeef884915b2e97b222ea68fc1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a21c21adc525485f809baf218d6a999c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7b7886aded7e47118d4dd14c0b88a894, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/27b46d7522244a28890e51138ce0c13b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/30dc9b5bce2344188a530598cb84da5c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a1b0fd0247644360bae6f3ea02c89c64, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68] to archive 2024-12-03T14:14:09,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:09,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7f8db4d1d6c44b1a8db51ba384bbf173 2024-12-03T14:14:09,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1e6b84f1748944c9abdd843a1bffbe30 2024-12-03T14:14:09,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/69d5043353ec4fa4a0bc548106d4ccfd 2024-12-03T14:14:09,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b5d1da2f1f114d87b689d4ad486c5333 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b5d1da2f1f114d87b689d4ad486c5333 2024-12-03T14:14:09,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9ae1f7263ccf4a6cbb778e87977df908 2024-12-03T14:14:09,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8d8b8e9b19f8462d8f975296a17d74fb 2024-12-03T14:14:09,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f2f49a2862974bab9b219115576b0b38 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f2f49a2862974bab9b219115576b0b38 2024-12-03T14:14:09,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/77f55be78f1747c6ac1f918ee51bedf7 2024-12-03T14:14:09,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55f91be7aae641609eec9c27820a6e91 2024-12-03T14:14:09,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/db7dd28928304509869b2910762176f9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/db7dd28928304509869b2910762176f9 2024-12-03T14:14:09,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/4819d4851a1344e6973c5fdb56d9dc96 2024-12-03T14:14:09,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/155383e81f9545a99dde5246b91f21bf 2024-12-03T14:14:09,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/79fd42a0214b44e58736a526172e8f99 2024-12-03T14:14:09,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/0f3298a063714c61ba0e6775ea67b6b6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/0f3298a063714c61ba0e6775ea67b6b6 2024-12-03T14:14:09,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a97222b7a1fe4146aaccb79a2cc29f5d 2024-12-03T14:14:09,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/721ff4ee22df44658c2932808be01d14 2024-12-03T14:14:09,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/ce9485eeef884915b2e97b222ea68fc1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/ce9485eeef884915b2e97b222ea68fc1 2024-12-03T14:14:09,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3aa78313ae3e49f5a6950d6fca717468 2024-12-03T14:14:09,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/573503d1e3514dbcae41d32bc119a897 2024-12-03T14:14:09,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/03e3f6a26e864e25b193945773f76fdc 2024-12-03T14:14:09,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a21c21adc525485f809baf218d6a999c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a21c21adc525485f809baf218d6a999c 2024-12-03T14:14:09,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/9f8522afa52b4997b25619bfac8cfcb9 2024-12-03T14:14:09,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/f077e32567464ac98c47869d70472384 2024-12-03T14:14:09,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7b7886aded7e47118d4dd14c0b88a894 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7b7886aded7e47118d4dd14c0b88a894 2024-12-03T14:14:09,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/3636e93bdd7c44479e2972b5bb8df4de 2024-12-03T14:14:09,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/b50b721d3ec54724bff07b4e315c21c5 2024-12-03T14:14:09,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e6e5ca759e9a47f3a2b97fa115d3ddd1 2024-12-03T14:14:09,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/27b46d7522244a28890e51138ce0c13b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/27b46d7522244a28890e51138ce0c13b 2024-12-03T14:14:09,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/7d87cdb649194df0a2b01c71da248a23 2024-12-03T14:14:09,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/c420cac11c4a417bad7970c28399b17b 2024-12-03T14:14:09,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/30dc9b5bce2344188a530598cb84da5c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/30dc9b5bce2344188a530598cb84da5c 2024-12-03T14:14:09,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/d0340efb11e14d2ca1abbfc0df2b2b88 2024-12-03T14:14:09,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/606032d827a74617bb442d9d9479a899 2024-12-03T14:14:09,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a1b0fd0247644360bae6f3ea02c89c64 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/a1b0fd0247644360bae6f3ea02c89c64 2024-12-03T14:14:09,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4fe4a6f9f3143fe97fe10cedadd1bf8 2024-12-03T14:14:09,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/55029f98dc574de5ac6723ad526f89d1 2024-12-03T14:14:09,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/e4a806c26a5649e797df5d9c28226b68 2024-12-03T14:14:09,901 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/recovered.edits/613.seqid, newMaxSeqId=613, maxSeqId=1 2024-12-03T14:14:09,902 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. 2024-12-03T14:14:09,902 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 8a9798ce308c36fb733235c6d0f2266b: Waiting for close lock at 1733235248390Running coprocessor pre-close hooks at 1733235248390Disabling compacts and flushes for region at 1733235248390Disabling writes for close at 1733235248391 (+1 ms)Obtaining lock to block concurrent updates at 1733235248391Preparing flush snapshotting stores in 8a9798ce308c36fb733235c6d0f2266b at 1733235248391Finished memstore snapshotting TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733235248391Flushing stores of TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b. at 1733235248392 (+1 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/A: creating writer at 1733235248392Flushing 8a9798ce308c36fb733235c6d0f2266b/A: appending metadata at 1733235248395 (+3 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/A: closing flushed file at 1733235248395Flushing 8a9798ce308c36fb733235c6d0f2266b/B: creating writer at 1733235248848 (+453 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/B: appending metadata at 1733235248853 (+5 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/B: closing flushed file at 1733235248853Flushing 8a9798ce308c36fb733235c6d0f2266b/C: creating writer at 1733235249263 (+410 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/C: appending metadata at 1733235249266 (+3 ms)Flushing 8a9798ce308c36fb733235c6d0f2266b/C: closing flushed file at 1733235249266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51ac1441: reopening flushed file at 1733235249683 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@484c46db: reopening flushed file at 1733235249691 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67de139a: reopening flushed file at 1733235249698 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 8a9798ce308c36fb733235c6d0f2266b in 1315ms, sequenceid=610, compaction requested=true at 1733235249707 (+9 ms)Writing region close event to WAL at 1733235249898 (+191 ms)Running coprocessor post-close hooks at 1733235249902 (+4 ms)Closed at 1733235249902 2024-12-03T14:14:09,905 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:09,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8a9798ce308c36fb733235c6d0f2266b, regionState=CLOSED 2024-12-03T14:14:09,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:09,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T14:14:09,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a9798ce308c36fb733235c6d0f2266b, server=3920e6a7c3b5,42177,1733235217010 in 1.6750 sec 2024-12-03T14:14:09,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T14:14:09,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a9798ce308c36fb733235c6d0f2266b, UNASSIGN in 1.6810 sec 2024-12-03T14:14:09,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T14:14:09,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6870 sec 2024-12-03T14:14:09,916 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235249916"}]},"ts":"1733235249916"} 2024-12-03T14:14:09,918 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:14:09,918 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:14:09,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7030 sec 2024-12-03T14:14:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T14:14:10,367 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:10,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:14:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,381 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T14:14:10,383 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,386 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:10,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/recovered.edits] 2024-12-03T14:14:10,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3800ec35f9204fe9b867cdb3c6615ec5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/3800ec35f9204fe9b867cdb3c6615ec5 2024-12-03T14:14:10,395 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8d491c2f2848499c8902a60e991fa911 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/8d491c2f2848499c8902a60e991fa911 2024-12-03T14:14:10,396 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/bf004550a2504f73a1b5fda661767255 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/A/bf004550a2504f73a1b5fda661767255 2024-12-03T14:14:10,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/684f1482b88f403ea8cba4b8b8d83df8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/684f1482b88f403ea8cba4b8b8d83df8 2024-12-03T14:14:10,400 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7d9163a77fac4f84a08ccf8b85adf9ea to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/7d9163a77fac4f84a08ccf8b85adf9ea 2024-12-03T14:14:10,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/c260f59e06664ce09a3813e6943aabe1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/B/c260f59e06664ce09a3813e6943aabe1 2024-12-03T14:14:10,404 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/13ef626c21d94b5bb40109da22a11791 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/13ef626c21d94b5bb40109da22a11791 2024-12-03T14:14:10,405 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1c87c5e6fd8347d2a0aaad8466a3c7d8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/1c87c5e6fd8347d2a0aaad8466a3c7d8 2024-12-03T14:14:10,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8a653feec2a647b4aa7fb6f939d107d8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/C/8a653feec2a647b4aa7fb6f939d107d8 2024-12-03T14:14:10,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/recovered.edits/613.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b/recovered.edits/613.seqid 2024-12-03T14:14:10,411 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/8a9798ce308c36fb733235c6d0f2266b 2024-12-03T14:14:10,411 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:14:10,416 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T14:14:10,424 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:14:10,457 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:14:10,459 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,460 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:14:10,460 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235250460"}]},"ts":"9223372036854775807"} 2024-12-03T14:14:10,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:14:10,464 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8a9798ce308c36fb733235c6d0f2266b, NAME => 'TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:14:10,465 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:14:10,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235250465"}]},"ts":"9223372036854775807"} 2024-12-03T14:14:10,469 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:14:10,470 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-12-03T14:14:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T14:14:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:14:10,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:10,496 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3920e6a7c3b5:42177-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_362070758_22 at /127.0.0.1:34490 [Waiting for operation #281] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_362070758_22 at /127.0.0.1:50142 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1738973269_22 at /127.0.0.1:49966 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 206) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10296 (was 10872) 2024-12-03T14:14:10,505 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=287, ProcessCount=11, AvailableMemoryMB=10295 2024-12-03T14:14:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:14:10,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:14:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:10,510 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:14:10,510 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:10,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-03T14:14:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:10,511 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:14:10,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741957_1133 (size=1061) 2024-12-03T14:14:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:10,921 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:14:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741958_1134 (size=53) 2024-12-03T14:14:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:11,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:14:11,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6bbe8ee77f3e0ec92707554117dec558, disabling compactions & flushes 2024-12-03T14:14:11,332 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. after waiting 0 ms 2024-12-03T14:14:11,332 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6bbe8ee77f3e0ec92707554117dec558: Waiting for close lock at 1733235251332Disabling compacts and flushes for region at 1733235251332Disabling writes for close at 1733235251332Writing region close event to WAL at 1733235251333 (+1 ms)Closed at 1733235251333 2024-12-03T14:14:11,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:14:11,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235251336"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235251336"}]},"ts":"1733235251336"} 2024-12-03T14:14:11,340 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:14:11,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:14:11,342 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235251342"}]},"ts":"1733235251342"} 2024-12-03T14:14:11,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:14:11,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, ASSIGN}] 2024-12-03T14:14:11,345 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, ASSIGN 2024-12-03T14:14:11,346 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:14:11,497 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:11,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, ASSIGN because future has completed 2024-12-03T14:14:11,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:11,664 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,664 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:14:11,665 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,665 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:14:11,665 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,665 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,666 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,668 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:11,669 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName A 2024-12-03T14:14:11,669 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:11,669 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:11,669 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,671 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:11,671 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName B 2024-12-03T14:14:11,671 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:11,672 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:11,672 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,673 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:11,673 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName C 2024-12-03T14:14:11,673 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:11,674 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:11,674 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,674 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,675 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,675 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,676 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,676 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,677 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:14:11,678 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,680 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:14:11,681 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 6bbe8ee77f3e0ec92707554117dec558; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60585380, jitterRate=-0.09720748662948608}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:14:11,681 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:11,681 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 6bbe8ee77f3e0ec92707554117dec558: Running coprocessor pre-open hook at 1733235251665Writing region info on filesystem at 1733235251665Initializing all the Stores at 1733235251666 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235251666Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235251666Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235251666Cleaning up temporary data from old regions at 1733235251676 (+10 ms)Running coprocessor post-open hooks at 1733235251681 (+5 ms)Region opened successfully at 1733235251681 2024-12-03T14:14:11,682 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., pid=32, masterSystemTime=1733235251655 2024-12-03T14:14:11,683 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,684 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:11,684 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:11,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:11,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T14:14:11,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 in 185 msec 2024-12-03T14:14:11,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-03T14:14:11,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, ASSIGN in 344 msec 2024-12-03T14:14:11,690 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:14:11,690 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235251690"}]},"ts":"1733235251690"} 2024-12-03T14:14:11,692 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:14:11,692 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:14:11,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-03T14:14:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-03T14:14:12,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:12,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:12,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:12,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:12,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3caff859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:12,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:12,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d767205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:12,677 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:12,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:12,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:14:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:14:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8f55055 2024-12-03T14:14:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:14:12,683 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:14:12,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:14:12,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:14:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:12,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741959_1135 (size=1097) 2024-12-03T14:14:13,109 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T14:14:13,109 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T14:14:13,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:14:13,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE}] 2024-12-03T14:14:13,124 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE 2024-12-03T14:14:13,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE because future has completed 2024-12-03T14:14:13,127 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:14:13,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:13,279 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,279 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:14:13,279 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 6bbe8ee77f3e0ec92707554117dec558, disabling compactions & flushes 2024-12-03T14:14:13,280 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. after waiting 0 ms 2024-12-03T14:14:13,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,285 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T14:14:13,286 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,286 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 6bbe8ee77f3e0ec92707554117dec558: Waiting for close lock at 1733235253279Running coprocessor pre-close hooks at 1733235253279Disabling compacts and flushes for region at 1733235253279Disabling writes for close at 1733235253280 (+1 ms)Writing region close event to WAL at 1733235253280Running coprocessor post-close hooks at 1733235253286 (+6 ms)Closed at 1733235253286 2024-12-03T14:14:13,286 WARN [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 6bbe8ee77f3e0ec92707554117dec558 to self. 2024-12-03T14:14:13,289 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,290 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=CLOSED 2024-12-03T14:14:13,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:13,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-03T14:14:13,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 in 167 msec 2024-12-03T14:14:13,298 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE; state=CLOSED, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=true 2024-12-03T14:14:13,449 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE because future has completed 2024-12-03T14:14:13,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:13,613 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,613 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:14:13,614 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,614 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:14:13,614 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,615 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,620 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,621 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:13,626 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName A 2024-12-03T14:14:13,628 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:13,629 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:13,629 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,630 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:13,630 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName B 2024-12-03T14:14:13,630 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:13,631 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:13,631 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,631 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:13,631 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bbe8ee77f3e0ec92707554117dec558 columnFamilyName C 2024-12-03T14:14:13,631 DEBUG [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:13,632 INFO [StoreOpener-6bbe8ee77f3e0ec92707554117dec558-1 {}] regionserver.HStore(327): Store=6bbe8ee77f3e0ec92707554117dec558/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:13,632 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,632 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,633 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,634 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,635 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,635 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,635 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:14:13,636 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,637 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6bbe8ee77f3e0ec92707554117dec558; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71280522, jitterRate=0.062162548303604126}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:14:13,637 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,639 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6bbe8ee77f3e0ec92707554117dec558: Running coprocessor pre-open hook at 1733235253615Writing region info on filesystem at 1733235253615Initializing all the Stores at 1733235253616 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235253616Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235253617 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235253617Cleaning up temporary data from old regions at 1733235253635 (+18 ms)Running coprocessor post-open hooks at 1733235253637 (+2 ms)Region opened successfully at 1733235253639 (+2 ms) 2024-12-03T14:14:13,639 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., pid=37, masterSystemTime=1733235253606 2024-12-03T14:14:13,641 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,641 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=OPEN, openSeqNum=5, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:13,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-03T14:14:13,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 in 192 msec 2024-12-03T14:14:13,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-03T14:14:13,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, REOPEN/MOVE in 523 msec 2024-12-03T14:14:13,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-03T14:14:13,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-12-03T14:14:13,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 961 msec 2024-12-03T14:14:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-03T14:14:13,657 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:13,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,659 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385878b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259191ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29edbc68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4a1050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b63ddc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3871d0c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344c8237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b1297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4743d3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4aa0e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,679 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abffbac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,680 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb8be74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e150354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,684 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f65cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e47e7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d96dc0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:13,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:13,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d12f83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:13,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:13,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c97c563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:13,694 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,694 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,694 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,694 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,695 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,695 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,696 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:13,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,697 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,698 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:13,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T14:14:13,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:13,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T14:14:13,701 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:13,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,702 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:13,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:13,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:14:13,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:14:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:13,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:13,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:13,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235313738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235313738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235313740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235313748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e4ea1aa092eb48b1b2b511b05854e301_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235253706/Put/seqid=0 2024-12-03T14:14:13,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235313751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741960_1136 (size=12154) 2024-12-03T14:14:13,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:13,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e4ea1aa092eb48b1b2b511b05854e301_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4ea1aa092eb48b1b2b511b05854e301_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:13,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/9dac9a60fff6434ea6ab65f987ec1962, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:13,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/9dac9a60fff6434ea6ab65f987ec1962 is 175, key is test_row_0/A:col10/1733235253706/Put/seqid=0 2024-12-03T14:14:13,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741961_1137 (size=30955) 2024-12-03T14:14:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T14:14:13,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/9dac9a60fff6434ea6ab65f987ec1962 2024-12-03T14:14:13,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T14:14:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:13,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235313856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235313856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235313856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235313856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235313866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:13,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a71ce853d9b14f799680cca413a3350a is 50, key is test_row_0/B:col10/1733235253706/Put/seqid=0 2024-12-03T14:14:13,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741962_1138 (size=12001) 2024-12-03T14:14:13,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a71ce853d9b14f799680cca413a3350a 2024-12-03T14:14:13,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/ad954e4b0c5d41938d400cfa2d72afa6 is 50, key is test_row_0/C:col10/1733235253706/Put/seqid=0 2024-12-03T14:14:13,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741963_1139 (size=12001) 2024-12-03T14:14:13,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/ad954e4b0c5d41938d400cfa2d72afa6 2024-12-03T14:14:13,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/9dac9a60fff6434ea6ab65f987ec1962 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962 2024-12-03T14:14:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962, entries=150, sequenceid=16, filesize=30.2 K 2024-12-03T14:14:13,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a71ce853d9b14f799680cca413a3350a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a 2024-12-03T14:14:13,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T14:14:13,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/ad954e4b0c5d41938d400cfa2d72afa6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6 2024-12-03T14:14:14,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T14:14:14,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6bbe8ee77f3e0ec92707554117dec558 in 289ms, sequenceid=16, compaction requested=false 2024-12-03T14:14:14,004 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T14:14:14,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T14:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,012 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T14:14:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T14:14:14,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033b9b4485e91641d0b1c1fcacb7cc10a7_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235253746/Put/seqid=0 2024-12-03T14:14:14,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741964_1140 (size=12154) 2024-12-03T14:14:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:14,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033b9b4485e91641d0b1c1fcacb7cc10a7_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033b9b4485e91641d0b1c1fcacb7cc10a7_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e8a29798c5df4959965a2281df158167, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e8a29798c5df4959965a2281df158167 is 175, key is test_row_0/A:col10/1733235253746/Put/seqid=0 2024-12-03T14:14:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741965_1141 (size=30955) 2024-12-03T14:14:14,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:14,074 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e8a29798c5df4959965a2281df158167 2024-12-03T14:14:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/90045bd539584e1b931e378efb3c96e1 is 50, key is test_row_0/B:col10/1733235253746/Put/seqid=0 2024-12-03T14:14:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741966_1142 (size=12001) 2024-12-03T14:14:14,128 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/90045bd539584e1b931e378efb3c96e1 2024-12-03T14:14:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/054f15262b5d480390bcbafc96dc1339 is 50, key is test_row_0/C:col10/1733235253746/Put/seqid=0 2024-12-03T14:14:14,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741967_1143 (size=12001) 2024-12-03T14:14:14,159 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/054f15262b5d480390bcbafc96dc1339 2024-12-03T14:14:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e8a29798c5df4959965a2281df158167 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167 2024-12-03T14:14:14,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167, entries=150, sequenceid=36, filesize=30.2 K 2024-12-03T14:14:14,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/90045bd539584e1b931e378efb3c96e1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1 2024-12-03T14:14:14,193 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T14:14:14,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/054f15262b5d480390bcbafc96dc1339 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339 2024-12-03T14:14:14,200 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T14:14:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6bbe8ee77f3e0ec92707554117dec558 in 190ms, sequenceid=36, compaction requested=false 2024-12-03T14:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T14:14:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T14:14:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:14:14,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T14:14:14,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 503 msec 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:14,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 514 msec 2024-12-03T14:14:14,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037e916bc37a4b420b911760f274711d6d_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741968_1144 (size=14594) 2024-12-03T14:14:14,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037e916bc37a4b420b911760f274711d6d_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037e916bc37a4b420b911760f274711d6d_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8b19907aabc34ab59cee114fe8e4dcb9, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8b19907aabc34ab59cee114fe8e4dcb9 is 175, key is test_row_0/A:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741969_1145 (size=39549) 2024-12-03T14:14:14,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8b19907aabc34ab59cee114fe8e4dcb9 2024-12-03T14:14:14,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/972b7099c0f94289a39024794295fdc0 is 50, key is test_row_0/B:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741970_1146 (size=12001) 2024-12-03T14:14:14,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/972b7099c0f94289a39024794295fdc0 2024-12-03T14:14:14,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/28839f4c8c354af78978dd164b3cc416 is 50, key is test_row_0/C:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T14:14:14,329 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:14,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T14:14:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T14:14:14,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:14,334 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:14,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741971_1147 (size=12001) 2024-12-03T14:14:14,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/28839f4c8c354af78978dd164b3cc416 2024-12-03T14:14:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8b19907aabc34ab59cee114fe8e4dcb9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9 2024-12-03T14:14:14,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9, entries=200, sequenceid=49, filesize=38.6 K 2024-12-03T14:14:14,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/972b7099c0f94289a39024794295fdc0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0 2024-12-03T14:14:14,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T14:14:14,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/28839f4c8c354af78978dd164b3cc416 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416 2024-12-03T14:14:14,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T14:14:14,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6bbe8ee77f3e0ec92707554117dec558 in 183ms, sequenceid=49, compaction requested=true 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:14,393 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:14,393 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:14,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:14,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:14,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:14,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=99.1 K 2024-12-03T14:14:14,396 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,396 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9] 2024-12-03T14:14:14,396 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:14,396 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:14,397 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,397 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.2 K 2024-12-03T14:14:14,397 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dac9a60fff6434ea6ab65f987ec1962, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733235253706 2024-12-03T14:14:14,398 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a71ce853d9b14f799680cca413a3350a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733235253706 2024-12-03T14:14:14,398 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a29798c5df4959965a2281df158167, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733235253726 2024-12-03T14:14:14,398 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 90045bd539584e1b931e378efb3c96e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733235253726 2024-12-03T14:14:14,399 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b19907aabc34ab59cee114fe8e4dcb9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:14,399 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 972b7099c0f94289a39024794295fdc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:14,413 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,427 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:14,428 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/19951f9708554d188d6a2f9b6c8b6d92 is 50, key is test_row_0/B:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,433 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120331e1fb9eccac422ca4b82d4ecc24a0c1_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T14:14:14,437 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120331e1fb9eccac422ca4b82d4ecc24a0c1_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,438 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120331e1fb9eccac422ca4b82d4ecc24a0c1_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741973_1149 (size=4469) 2024-12-03T14:14:14,477 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#129 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:14,478 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8643b1e22077498da41581e22c6079d5 is 175, key is test_row_0/A:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741972_1148 (size=12104) 2024-12-03T14:14:14,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T14:14:14,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f3c20a59fe6748c3a28892c97005a9c6_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235254238/Put/seqid=0 2024-12-03T14:14:14,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741974_1150 (size=31058) 2024-12-03T14:14:14,525 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8643b1e22077498da41581e22c6079d5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5 2024-12-03T14:14:14,533 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 8643b1e22077498da41581e22c6079d5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:14,533 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,533 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235254393; duration=0sec 2024-12-03T14:14:14,533 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:14,533 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:14,533 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:14,534 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:14,534 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:14,534 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,535 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.2 K 2024-12-03T14:14:14,538 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad954e4b0c5d41938d400cfa2d72afa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733235253706 2024-12-03T14:14:14,538 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054f15262b5d480390bcbafc96dc1339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733235253726 2024-12-03T14:14:14,539 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28839f4c8c354af78978dd164b3cc416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:14,548 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:14,548 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/417c20daabd64673928975b11ffb1a4b is 50, key is test_row_0/C:col10/1733235254093/Put/seqid=0 2024-12-03T14:14:14,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741975_1151 (size=12154) 2024-12-03T14:14:14,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:14,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,560 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f3c20a59fe6748c3a28892c97005a9c6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3c20a59fe6748c3a28892c97005a9c6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/681ff7859bc44ed19d4993f7ace89664, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/681ff7859bc44ed19d4993f7ace89664 is 175, key is test_row_0/A:col10/1733235254238/Put/seqid=0 2024-12-03T14:14:14,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741976_1152 (size=12104) 2024-12-03T14:14:14,611 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/417c20daabd64673928975b11ffb1a4b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/417c20daabd64673928975b11ffb1a4b 2024-12-03T14:14:14,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741977_1153 (size=30955) 2024-12-03T14:14:14,615 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/681ff7859bc44ed19d4993f7ace89664 2024-12-03T14:14:14,622 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into 417c20daabd64673928975b11ffb1a4b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:14,622 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,622 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235254393; duration=0sec 2024-12-03T14:14:14,622 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:14,622 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:14,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/063d7464b30a4eee96e28b852b5c3710 is 50, key is test_row_0/B:col10/1733235254238/Put/seqid=0 2024-12-03T14:14:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T14:14:14,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741978_1154 (size=12001) 2024-12-03T14:14:14,654 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/063d7464b30a4eee96e28b852b5c3710 2024-12-03T14:14:14,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/197b74e584384ae9ad0b965a6fa80f35 is 50, key is test_row_0/C:col10/1733235254238/Put/seqid=0 2024-12-03T14:14:14,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741979_1155 (size=12001) 2024-12-03T14:14:14,687 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/197b74e584384ae9ad0b965a6fa80f35 2024-12-03T14:14:14,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/681ff7859bc44ed19d4993f7ace89664 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664 2024-12-03T14:14:14,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,705 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664, entries=150, sequenceid=68, filesize=30.2 K 2024-12-03T14:14:14,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/063d7464b30a4eee96e28b852b5c3710 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710 2024-12-03T14:14:14,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T14:14:14,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/197b74e584384ae9ad0b965a6fa80f35 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35 2024-12-03T14:14:14,729 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35, entries=150, sequenceid=68, filesize=11.7 K 2024-12-03T14:14:14,730 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6bbe8ee77f3e0ec92707554117dec558 in 242ms, sequenceid=68, compaction requested=false 2024-12-03T14:14:14,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:14,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T14:14:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T14:14:14,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T14:14:14,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 398 msec 2024-12-03T14:14:14,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 406 msec 2024-12-03T14:14:14,886 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/19951f9708554d188d6a2f9b6c8b6d92 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19951f9708554d188d6a2f9b6c8b6d92 2024-12-03T14:14:14,893 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into 19951f9708554d188d6a2f9b6c8b6d92(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:14,893 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:14,893 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235254393; duration=0sec 2024-12-03T14:14:14,893 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:14,893 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:14,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:14:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:14,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:14,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cfd12ef41ed4bd78fa234b5c4beb70e_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235314931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235314934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235314939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235314940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235314945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T14:14:14,957 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:14,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T14:14:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:14,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:14,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741980_1156 (size=12154) 2024-12-03T14:14:14,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:14,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:14,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:14,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cfd12ef41ed4bd78fa234b5c4beb70e_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cfd12ef41ed4bd78fa234b5c4beb70e_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:14,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74f940b9c12240d8a707ed38cb34ee57, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:14,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74f940b9c12240d8a707ed38cb34ee57 is 175, key is test_row_0/A:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:14,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741981_1157 (size=30955) 2024-12-03T14:14:14,992 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74f940b9c12240d8a707ed38cb34ee57 2024-12-03T14:14:15,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a08ea8cdf84b4ac1881775ac3428cd2c is 50, key is test_row_0/B:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:15,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741982_1158 (size=12001) 2024-12-03T14:14:15,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a08ea8cdf84b4ac1881775ac3428cd2c 2024-12-03T14:14:15,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6215838bb01b4c9cb731a64ef6a3e7c1 is 50, key is test_row_0/C:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235315047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235315047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235315047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235315047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235315056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741983_1159 (size=12001) 2024-12-03T14:14:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:15,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235315256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235315256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235315256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235315256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235315266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:15,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,445 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:14:15,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6215838bb01b4c9cb731a64ef6a3e7c1 2024-12-03T14:14:15,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74f940b9c12240d8a707ed38cb34ee57 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57 2024-12-03T14:14:15,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57, entries=150, sequenceid=86, filesize=30.2 K 2024-12-03T14:14:15,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a08ea8cdf84b4ac1881775ac3428cd2c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c 2024-12-03T14:14:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T14:14:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6215838bb01b4c9cb731a64ef6a3e7c1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1 2024-12-03T14:14:15,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T14:14:15,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6bbe8ee77f3e0ec92707554117dec558 in 620ms, sequenceid=86, compaction requested=true 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:15,520 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:15,520 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:15,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:15,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:15,521 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:15,521 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:15,521 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:15,521 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:15,522 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,522 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,522 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19951f9708554d188d6a2f9b6c8b6d92, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.3 K 2024-12-03T14:14:15,522 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=90.8 K 2024-12-03T14:14:15,522 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,522 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57] 2024-12-03T14:14:15,522 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 19951f9708554d188d6a2f9b6c8b6d92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:15,522 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8643b1e22077498da41581e22c6079d5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:15,522 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 063d7464b30a4eee96e28b852b5c3710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733235254224 2024-12-03T14:14:15,523 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681ff7859bc44ed19d4993f7ace89664, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733235254224 2024-12-03T14:14:15,523 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a08ea8cdf84b4ac1881775ac3428cd2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:15,523 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f940b9c12240d8a707ed38cb34ee57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:15,532 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:15,533 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/eb99349f8f054447bf9c249cd7c77b91 is 50, key is test_row_0/B:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:15,535 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:15,539 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412035f43979bbb184c7d8ed34f9393477517_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:15,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741984_1160 (size=12207) 2024-12-03T14:14:15,542 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412035f43979bbb184c7d8ed34f9393477517_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:15,542 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035f43979bbb184c7d8ed34f9393477517_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:15,547 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/eb99349f8f054447bf9c249cd7c77b91 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/eb99349f8f054447bf9c249cd7c77b91 2024-12-03T14:14:15,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741985_1161 (size=4469) 2024-12-03T14:14:15,554 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into eb99349f8f054447bf9c249cd7c77b91(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:15,554 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:15,554 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235255520; duration=0sec 2024-12-03T14:14:15,554 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:15,554 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:15,554 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:15,555 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:15,555 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:15,555 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,555 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/417c20daabd64673928975b11ffb1a4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.3 K 2024-12-03T14:14:15,556 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 417c20daabd64673928975b11ffb1a4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235254087 2024-12-03T14:14:15,556 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 197b74e584384ae9ad0b965a6fa80f35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733235254224 2024-12-03T14:14:15,557 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6215838bb01b4c9cb731a64ef6a3e7c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:15,567 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:15,567 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/72202d608d434cdcac6d851524e83b0d is 50, key is test_row_0/C:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:15,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741986_1162 (size=12207) 2024-12-03T14:14:15,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120327f96c572ce14b82b6ed5bf2685bdfd6_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235254932/Put/seqid=0 2024-12-03T14:14:15,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/72202d608d434cdcac6d851524e83b0d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/72202d608d434cdcac6d851524e83b0d 2024-12-03T14:14:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:15,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235315585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235315588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235315589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235315590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235315590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,596 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into 72202d608d434cdcac6d851524e83b0d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:15,596 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:15,596 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235255520; duration=0sec 2024-12-03T14:14:15,596 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:15,596 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741987_1163 (size=12154) 2024-12-03T14:14:15,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:15,627 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120327f96c572ce14b82b6ed5bf2685bdfd6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120327f96c572ce14b82b6ed5bf2685bdfd6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:15,628 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c1f86b62968e4b1b9faac3120de109b5, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:15,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c1f86b62968e4b1b9faac3120de109b5 is 175, key is test_row_0/A:col10/1733235254932/Put/seqid=0 2024-12-03T14:14:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741988_1164 (size=30955) 2024-12-03T14:14:15,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c1f86b62968e4b1b9faac3120de109b5 2024-12-03T14:14:15,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/85bedd175107455d895776c3ad7ef3f1 is 50, key is test_row_0/B:col10/1733235254932/Put/seqid=0 2024-12-03T14:14:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741989_1165 (size=12001) 2024-12-03T14:14:15,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/85bedd175107455d895776c3ad7ef3f1 2024-12-03T14:14:15,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5035ccdbcb68442987252490d29d043d is 50, key is test_row_0/C:col10/1733235254932/Put/seqid=0 2024-12-03T14:14:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741990_1166 (size=12001) 2024-12-03T14:14:15,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5035ccdbcb68442987252490d29d043d 2024-12-03T14:14:15,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c1f86b62968e4b1b9faac3120de109b5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5 2024-12-03T14:14:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235315696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235315696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235315696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5, entries=150, sequenceid=106, filesize=30.2 K 2024-12-03T14:14:15,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235315706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/85bedd175107455d895776c3ad7ef3f1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1 2024-12-03T14:14:15,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235315707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1, entries=150, sequenceid=106, filesize=11.7 K 2024-12-03T14:14:15,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5035ccdbcb68442987252490d29d043d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d 2024-12-03T14:14:15,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d, entries=150, sequenceid=106, filesize=11.7 K 2024-12-03T14:14:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:15,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6bbe8ee77f3e0ec92707554117dec558 in 164ms, sequenceid=106, compaction requested=false 2024-12-03T14:14:15,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:15,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T14:14:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:15,887 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:14:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:15,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c88119717e342ea9319336d5db2e7b0_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235255584/Put/seqid=0 2024-12-03T14:14:15,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741991_1167 (size=12154) 2024-12-03T14:14:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:15,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235315934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235315935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235315936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235315942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235315944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:15,951 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:15,952 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/344ca28e69554cd08837f1ec66d99930 is 175, key is test_row_0/A:col10/1733235254899/Put/seqid=0 2024-12-03T14:14:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741992_1168 (size=31161) 2024-12-03T14:14:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235316047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235316047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235316047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235316056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235316057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:16,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235316256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235316256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235316256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235316266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235316267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:16,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035c88119717e342ea9319336d5db2e7b0_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c88119717e342ea9319336d5db2e7b0_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:16,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74c7fa6ec4b941cb8931ada6529f2a17, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:16,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74c7fa6ec4b941cb8931ada6529f2a17 is 175, key is test_row_0/A:col10/1733235255584/Put/seqid=0 2024-12-03T14:14:16,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741993_1169 (size=30955) 2024-12-03T14:14:16,320 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74c7fa6ec4b941cb8931ada6529f2a17 2024-12-03T14:14:16,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b4bc439c968e434c8ca2dbdc23539d08 is 50, key is test_row_0/B:col10/1733235255584/Put/seqid=0 2024-12-03T14:14:16,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741994_1170 (size=12001) 2024-12-03T14:14:16,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b4bc439c968e434c8ca2dbdc23539d08 2024-12-03T14:14:16,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/979ee46ae35c4bcc88755375f8bc5f83 is 50, key is test_row_0/C:col10/1733235255584/Put/seqid=0 2024-12-03T14:14:16,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741995_1171 (size=12001) 2024-12-03T14:14:16,371 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/344ca28e69554cd08837f1ec66d99930 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930 2024-12-03T14:14:16,381 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 344ca28e69554cd08837f1ec66d99930(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:16,381 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:16,381 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235255520; duration=0sec 2024-12-03T14:14:16,382 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:16,382 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:16,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235316566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235316566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235316566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235316576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235316576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/979ee46ae35c4bcc88755375f8bc5f83 2024-12-03T14:14:16,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/74c7fa6ec4b941cb8931ada6529f2a17 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17 2024-12-03T14:14:16,767 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17, entries=150, sequenceid=120, filesize=30.2 K 2024-12-03T14:14:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b4bc439c968e434c8ca2dbdc23539d08 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08 2024-12-03T14:14:16,773 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08, entries=150, sequenceid=120, filesize=11.7 K 2024-12-03T14:14:16,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/979ee46ae35c4bcc88755375f8bc5f83 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83 2024-12-03T14:14:16,779 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83, entries=150, sequenceid=120, filesize=11.7 K 2024-12-03T14:14:16,780 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6bbe8ee77f3e0ec92707554117dec558 in 893ms, sequenceid=120, compaction requested=true 2024-12-03T14:14:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T14:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T14:14:16,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T14:14:16,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-12-03T14:14:16,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8260 sec 2024-12-03T14:14:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235255928, max timestamp=1733235257076Num uniques 859; 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235255928, max timestamp=1733235257076Num uniques 859; 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235255928, max timestamp=1733235257076Num uniques 859; 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:17,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:17,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cdfd34fae63c408cbd0b77b91462bdab_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741996_1172 (size=17284) 2024-12-03T14:14:17,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T14:14:17,107 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:17,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T14:14:17,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235257098, max timestamp=1733235257107Num uniques 859; 2024-12-03T14:14:17,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:17,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235257098, max timestamp=1733235257108Num uniques 859; 2024-12-03T14:14:17,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235257098, max timestamp=1733235257108Num uniques 859; 2024-12-03T14:14:17,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:17,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:17,112 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:17,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:17,113 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cdfd34fae63c408cbd0b77b91462bdab_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cdfd34fae63c408cbd0b77b91462bdab_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:17,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:17,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/759526c5614e4521ac2fcd82dc80f19f, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/759526c5614e4521ac2fcd82dc80f19f is 175, key is test_row_0/A:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733235257123, max timestamp=1733235257145Num uniques 1068; 2024-12-03T14:14:17,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733235257123, max timestamp=1733235257145Num uniques 1068; 2024-12-03T14:14:17,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-12-03T14:14:17,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-12-03T14:14:17,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733235257123, max timestamp=1733235257145Num uniques 1068; 2024-12-03T14:14:17,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-12-03T14:14:17,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741997_1173 (size=48389) 2024-12-03T14:14:17,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/759526c5614e4521ac2fcd82dc80f19f 2024-12-03T14:14:17,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/0502dacfe83740cabb501495de77cd77 is 50, key is test_row_0/B:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235317173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235317175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235317175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235317176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235317182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741998_1174 (size=12151) 2024-12-03T14:14:17,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/0502dacfe83740cabb501495de77cd77 2024-12-03T14:14:17,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0daeb97241ef4d0eb98944e095d5f2ea is 50, key is test_row_0/C:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741999_1175 (size=12151) 2024-12-03T14:14:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:17,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T14:14:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235317286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235317296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235317296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235317296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235317296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T14:14:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235317496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235317506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235317506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235317506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235317506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T14:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:17,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0daeb97241ef4d0eb98944e095d5f2ea 2024-12-03T14:14:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/759526c5614e4521ac2fcd82dc80f19f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f 2024-12-03T14:14:17,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f, entries=250, sequenceid=143, filesize=47.3 K 2024-12-03T14:14:17,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/0502dacfe83740cabb501495de77cd77 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77 2024-12-03T14:14:17,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T14:14:17,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0daeb97241ef4d0eb98944e095d5f2ea as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea 2024-12-03T14:14:17,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea, entries=150, sequenceid=143, filesize=11.9 K 2024-12-03T14:14:17,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for 6bbe8ee77f3e0ec92707554117dec558 in 564ms, sequenceid=143, compaction requested=true 2024-12-03T14:14:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:17,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:17,643 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:17,643 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:17,646 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:17,646 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:17,646 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,646 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/eb99349f8f054447bf9c249cd7c77b91, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=47.2 K 2024-12-03T14:14:17,647 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:17,647 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:17,647 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,647 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=138.1 K 2024-12-03T14:14:17,647 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,647 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f] 2024-12-03T14:14:17,648 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting eb99349f8f054447bf9c249cd7c77b91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:17,648 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 344ca28e69554cd08837f1ec66d99930, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:17,648 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 85bedd175107455d895776c3ad7ef3f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733235254928 2024-12-03T14:14:17,648 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f86b62968e4b1b9faac3120de109b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733235254928 2024-12-03T14:14:17,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b4bc439c968e434c8ca2dbdc23539d08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733235255584 2024-12-03T14:14:17,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74c7fa6ec4b941cb8931ada6529f2a17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733235255584 2024-12-03T14:14:17,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0502dacfe83740cabb501495de77cd77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255941 2024-12-03T14:14:17,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 759526c5614e4521ac2fcd82dc80f19f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255928 2024-12-03T14:14:17,669 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,674 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:17,675 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/d596b097d1ac4f358b983fb70793b6f4 is 50, key is test_row_0/B:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,682 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a7674480cf0449c0b857636dc1dce98f_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,684 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a7674480cf0449c0b857636dc1dce98f_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,684 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a7674480cf0449c0b857636dc1dce98f_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742000_1176 (size=12493) 2024-12-03T14:14:17,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742001_1177 (size=4469) 2024-12-03T14:14:17,696 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/d596b097d1ac4f358b983fb70793b6f4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/d596b097d1ac4f358b983fb70793b6f4 2024-12-03T14:14:17,702 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into d596b097d1ac4f358b983fb70793b6f4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:17,702 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:17,702 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=12, startTime=1733235257643; duration=0sec 2024-12-03T14:14:17,702 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:17,702 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:17,702 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:17,703 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:17,703 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:17,703 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,703 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/72202d608d434cdcac6d851524e83b0d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=47.2 K 2024-12-03T14:14:17,704 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 72202d608d434cdcac6d851524e83b0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235254572 2024-12-03T14:14:17,707 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5035ccdbcb68442987252490d29d043d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733235254928 2024-12-03T14:14:17,708 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 979ee46ae35c4bcc88755375f8bc5f83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733235255584 2024-12-03T14:14:17,708 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0daeb97241ef4d0eb98944e095d5f2ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255941 2024-12-03T14:14:17,719 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:17,719 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/b5c393ae6edb43fd9d797f5e6f5eec34 is 50, key is test_row_0/C:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:17,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742002_1178 (size=12493) 2024-12-03T14:14:17,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:17,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:17,733 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/b5c393ae6edb43fd9d797f5e6f5eec34 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b5c393ae6edb43fd9d797f5e6f5eec34 2024-12-03T14:14:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036719ac4311c94703b6164baf17c390b5_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235257179/Put/seqid=0 2024-12-03T14:14:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:17,744 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into b5c393ae6edb43fd9d797f5e6f5eec34(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:17,745 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:17,745 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=12, startTime=1733235257645; duration=0sec 2024-12-03T14:14:17,745 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:17,745 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:17,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742003_1179 (size=12304) 2024-12-03T14:14:17,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:17,755 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036719ac4311c94703b6164baf17c390b5_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036719ac4311c94703b6164baf17c390b5_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e1d1625a2cf3499f9e0447b74b46a38b, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e1d1625a2cf3499f9e0447b74b46a38b is 175, key is test_row_0/A:col10/1733235257179/Put/seqid=0 2024-12-03T14:14:17,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742004_1180 (size=31105) 2024-12-03T14:14:17,762 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e1d1625a2cf3499f9e0447b74b46a38b 2024-12-03T14:14:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/3c26d91de02a48e0a25c6c9ed9d98a7b is 50, key is test_row_0/B:col10/1733235257179/Put/seqid=0 2024-12-03T14:14:17,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742005_1181 (size=12151) 2024-12-03T14:14:17,800 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/3c26d91de02a48e0a25c6c9ed9d98a7b 2024-12-03T14:14:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:17,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/52c60350e20a4c328b0a54219c03161e is 50, key is test_row_0/C:col10/1733235257179/Put/seqid=0 2024-12-03T14:14:17,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235317816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235317816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235317818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235317820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235317820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742006_1182 (size=12151) 2024-12-03T14:14:17,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235317926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235317935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235317936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,096 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:18,096 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/d43fd3b2704a4b6098910918d652cd27 is 175, key is test_row_0/A:col10/1733235257076/Put/seqid=0 2024-12-03T14:14:18,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742007_1183 (size=31447) 2024-12-03T14:14:18,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235318136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235318146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235318146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,230 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/52c60350e20a4c328b0a54219c03161e 2024-12-03T14:14:18,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e1d1625a2cf3499f9e0447b74b46a38b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b 2024-12-03T14:14:18,240 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b, entries=150, sequenceid=198, filesize=30.4 K 2024-12-03T14:14:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/3c26d91de02a48e0a25c6c9ed9d98a7b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b 2024-12-03T14:14:18,244 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T14:14:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/52c60350e20a4c328b0a54219c03161e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e 2024-12-03T14:14:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:18,250 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e, entries=150, sequenceid=198, filesize=11.9 K 2024-12-03T14:14:18,251 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=33.54 KB/34350 for 6bbe8ee77f3e0ec92707554117dec558 in 525ms, sequenceid=198, compaction requested=false 2024-12-03T14:14:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T14:14:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T14:14:18,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T14:14:18,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-12-03T14:14:18,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1450 sec 2024-12-03T14:14:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:18,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:18,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120378cfafdefb0d41ed81a3e7df6365c616_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:18,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742008_1184 (size=12304) 2024-12-03T14:14:18,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235258353, max timestamp=1733235258362Num uniques 859; 2024-12-03T14:14:18,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:18,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235258353, max timestamp=1733235258362Num uniques 859; 2024-12-03T14:14:18,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:18,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235258353, max timestamp=1733235258362Num uniques 859; 2024-12-03T14:14:18,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:18,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235318405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235318405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235318446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235318456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235318456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,509 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/d43fd3b2704a4b6098910918d652cd27 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27 2024-12-03T14:14:18,514 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into d43fd3b2704a4b6098910918d652cd27(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:18,514 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:18,514 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=12, startTime=1733235257643; duration=0sec 2024-12-03T14:14:18,514 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:18,514 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235318516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235318516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235318726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235318726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,745 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:18,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120378cfafdefb0d41ed81a3e7df6365c616_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378cfafdefb0d41ed81a3e7df6365c616_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:18,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/51616e51c4154fe8a0fdbe79428270e6, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:18,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/51616e51c4154fe8a0fdbe79428270e6 is 175, key is test_row_0/A:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:18,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742009_1185 (size=31105) 2024-12-03T14:14:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235318956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235318966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235318966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235319035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235319036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/51616e51c4154fe8a0fdbe79428270e6 2024-12-03T14:14:19,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/bfa7706f9dfa4794bc78585995f996f0 is 50, key is test_row_0/B:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:19,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742010_1186 (size=12151) 2024-12-03T14:14:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T14:14:19,266 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:19,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T14:14:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:19,268 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:19,269 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:19,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:19,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T14:14:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235319546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235319546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/bfa7706f9dfa4794bc78585995f996f0 2024-12-03T14:14:19,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T14:14:19,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:19,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/2fc0b94764054e5fbf2b31d7d198f33c is 50, key is test_row_0/C:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:19,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742011_1187 (size=12151) 2024-12-03T14:14:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:19,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T14:14:19,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:19,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T14:14:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:19,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235319966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,967 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235319976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,981 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:19,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/2fc0b94764054e5fbf2b31d7d198f33c 2024-12-03T14:14:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235319986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:19,988 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:19,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/51616e51c4154fe8a0fdbe79428270e6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6 2024-12-03T14:14:20,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6, entries=150, sequenceid=209, filesize=30.4 K 2024-12-03T14:14:20,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/bfa7706f9dfa4794bc78585995f996f0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0 2024-12-03T14:14:20,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T14:14:20,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/2fc0b94764054e5fbf2b31d7d198f33c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c 2024-12-03T14:14:20,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c, entries=150, sequenceid=209, filesize=11.9 K 2024-12-03T14:14:20,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 6bbe8ee77f3e0ec92707554117dec558 in 1697ms, sequenceid=209, compaction requested=true 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:20,025 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:20,025 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:20,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:20,027 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:20,027 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:20,027 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:20,027 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:20,027 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:20,027 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:20,027 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/d596b097d1ac4f358b983fb70793b6f4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.9 K 2024-12-03T14:14:20,028 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=91.5 K 2024-12-03T14:14:20,028 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:20,028 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6] 2024-12-03T14:14:20,028 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43fd3b2704a4b6098910918d652cd27, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255941 2024-12-03T14:14:20,028 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d596b097d1ac4f358b983fb70793b6f4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255941 2024-12-03T14:14:20,028 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c26d91de02a48e0a25c6c9ed9d98a7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733235257170 2024-12-03T14:14:20,028 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1d1625a2cf3499f9e0447b74b46a38b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733235257170 2024-12-03T14:14:20,029 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa7706f9dfa4794bc78585995f996f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:20,029 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51616e51c4154fe8a0fdbe79428270e6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:20,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T14:14:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:20,034 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T14:14:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:20,057 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:20,058 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/f29d3894b1ef499c91ad6a8cced0985d is 50, key is test_row_0/B:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:20,074 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:20,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a0435aa16db94b10bb52112063d09824_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235258403/Put/seqid=0 2024-12-03T14:14:20,096 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e3be04aa715e466c9ec5b7cb9a257eaf_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:20,098 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e3be04aa715e466c9ec5b7cb9a257eaf_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:20,098 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e3be04aa715e466c9ec5b7cb9a257eaf_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:20,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742012_1188 (size=12595) 2024-12-03T14:14:20,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742013_1189 (size=12304) 2024-12-03T14:14:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742014_1190 (size=4469) 2024-12-03T14:14:20,179 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#160 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:20,180 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/f9678e485026408fbbf9cb89408e179a is 175, key is test_row_0/A:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:20,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742015_1191 (size=31549) 2024-12-03T14:14:20,197 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/f9678e485026408fbbf9cb89408e179a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a 2024-12-03T14:14:20,213 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into f9678e485026408fbbf9cb89408e179a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:20,213 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:20,213 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235260025; duration=0sec 2024-12-03T14:14:20,216 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:20,216 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:20,216 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:20,219 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:20,219 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:20,220 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:20,220 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b5c393ae6edb43fd9d797f5e6f5eec34, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=35.9 K 2024-12-03T14:14:20,222 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c393ae6edb43fd9d797f5e6f5eec34, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1733235255941 2024-12-03T14:14:20,223 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c60350e20a4c328b0a54219c03161e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733235257170 2024-12-03T14:14:20,224 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc0b94764054e5fbf2b31d7d198f33c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:20,238 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:20,239 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/b598d2f5513e47488cf0a05b7ed37985 is 50, key is test_row_0/C:col10/1733235257819/Put/seqid=0 2024-12-03T14:14:20,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742016_1192 (size=12595) 2024-12-03T14:14:20,284 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/b598d2f5513e47488cf0a05b7ed37985 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b598d2f5513e47488cf0a05b7ed37985 2024-12-03T14:14:20,293 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into b598d2f5513e47488cf0a05b7ed37985(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:20,293 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:20,293 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235260025; duration=0sec 2024-12-03T14:14:20,293 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:20,293 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:20,538 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/f29d3894b1ef499c91ad6a8cced0985d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/f29d3894b1ef499c91ad6a8cced0985d 2024-12-03T14:14:20,543 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into f29d3894b1ef499c91ad6a8cced0985d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:20,543 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:20,543 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235260025; duration=0sec 2024-12-03T14:14:20,543 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:20,543 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:20,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a0435aa16db94b10bb52112063d09824_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a0435aa16db94b10bb52112063d09824_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:20,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8063d63a6afa4ae4b62ddfc3336cc788, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:20,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8063d63a6afa4ae4b62ddfc3336cc788 is 175, key is test_row_0/A:col10/1733235258403/Put/seqid=0 2024-12-03T14:14:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:20,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:20,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742017_1193 (size=31105) 2024-12-03T14:14:20,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235320588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235320588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235320696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235320697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235320906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235320906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:20,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:20,964 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8063d63a6afa4ae4b62ddfc3336cc788 2024-12-03T14:14:20,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/28a2c2928bed440cb9f6aecb51ac4ba3 is 50, key is test_row_0/B:col10/1733235258403/Put/seqid=0 2024-12-03T14:14:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742018_1194 (size=12151) 2024-12-03T14:14:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235321216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235321216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,381 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/28a2c2928bed440cb9f6aecb51ac4ba3 2024-12-03T14:14:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/040708b48bd044638a3d14fb896f3a82 is 50, key is test_row_0/C:col10/1733235258403/Put/seqid=0 2024-12-03T14:14:21,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742019_1195 (size=12151) 2024-12-03T14:14:21,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/040708b48bd044638a3d14fb896f3a82 2024-12-03T14:14:21,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/8063d63a6afa4ae4b62ddfc3336cc788 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788 2024-12-03T14:14:21,401 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788, entries=150, sequenceid=247, filesize=30.4 K 2024-12-03T14:14:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/28a2c2928bed440cb9f6aecb51ac4ba3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3 2024-12-03T14:14:21,406 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T14:14:21,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/040708b48bd044638a3d14fb896f3a82 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82 2024-12-03T14:14:21,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82, entries=150, sequenceid=247, filesize=11.9 K 2024-12-03T14:14:21,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 6bbe8ee77f3e0ec92707554117dec558 in 1378ms, sequenceid=247, compaction requested=false 2024-12-03T14:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T14:14:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T14:14:21,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T14:14:21,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1440 sec 2024-12-03T14:14:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:21,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1490 sec 2024-12-03T14:14:21,427 INFO [master/3920e6a7c3b5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T14:14:21,427 INFO [master/3920e6a7c3b5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T14:14:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:21,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:14:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:21,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:21,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030e48c22cb3b04df5a37d53660bb44cd0_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742020_1196 (size=17534) 2024-12-03T14:14:21,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:21,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030e48c22cb3b04df5a37d53660bb44cd0_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e48c22cb3b04df5a37d53660bb44cd0_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:21,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/df5613b7b3524808896ab1f7daae64b7, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:21,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/df5613b7b3524808896ab1f7daae64b7 is 175, key is test_row_0/A:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:21,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742021_1197 (size=48639) 2024-12-03T14:14:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235321758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235321758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235321866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235321866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:21,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235321986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:21,989 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235322005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235322006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,006 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,007 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235322076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235322076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/df5613b7b3524808896ab1f7daae64b7 2024-12-03T14:14:22,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/48d74826bc754fca97dd1e208d856c07 is 50, key is test_row_0/B:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742022_1198 (size=12301) 2024-12-03T14:14:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235322386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235322386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/48d74826bc754fca97dd1e208d856c07 2024-12-03T14:14:22,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/1f3df6d8b1e648e58bb2cd270010a81d is 50, key is test_row_0/C:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:22,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742023_1199 (size=12301) 2024-12-03T14:14:22,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235322896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235322896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:22,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/1f3df6d8b1e648e58bb2cd270010a81d 2024-12-03T14:14:22,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/df5613b7b3524808896ab1f7daae64b7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7 2024-12-03T14:14:22,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7, entries=250, sequenceid=262, filesize=47.5 K 2024-12-03T14:14:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/48d74826bc754fca97dd1e208d856c07 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07 2024-12-03T14:14:22,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T14:14:22,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/1f3df6d8b1e648e58bb2cd270010a81d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d 2024-12-03T14:14:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T14:14:23,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6bbe8ee77f3e0ec92707554117dec558 in 1273ms, sequenceid=262, compaction requested=true 2024-12-03T14:14:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:23,000 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:23,000 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:23,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:23,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:23,001 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:23,001 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:23,001 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:23,001 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:23,001 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:23,001 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:23,001 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=108.7 K 2024-12-03T14:14:23,001 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/f29d3894b1ef499c91ad6a8cced0985d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.2 K 2024-12-03T14:14:23,001 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:23,001 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7] 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f29d3894b1ef499c91ad6a8cced0985d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9678e485026408fbbf9cb89408e179a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 28a2c2928bed440cb9f6aecb51ac4ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733235258391 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8063d63a6afa4ae4b62ddfc3336cc788, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733235258391 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d74826bc754fca97dd1e208d856c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:23,002 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5613b7b3524808896ab1f7daae64b7, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:23,011 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:23,012 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:23,012 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/e4455634825e4529947e2e84f07789ac is 50, key is test_row_0/B:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:23,018 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f1a43c7fa06447d09b49086cc4103c30_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:23,019 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f1a43c7fa06447d09b49086cc4103c30_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:23,019 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f1a43c7fa06447d09b49086cc4103c30_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742024_1200 (size=12847) 2024-12-03T14:14:23,026 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/e4455634825e4529947e2e84f07789ac as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/e4455634825e4529947e2e84f07789ac 2024-12-03T14:14:23,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742025_1201 (size=4469) 2024-12-03T14:14:23,033 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#169 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:23,033 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into e4455634825e4529947e2e84f07789ac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:23,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:23,033 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235263000; duration=0sec 2024-12-03T14:14:23,033 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2772b1405e5141a9859741e3f4465b34 is 175, key is test_row_0/A:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:23,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:23,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:23,034 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:23,035 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:23,035 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:23,035 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:23,035 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b598d2f5513e47488cf0a05b7ed37985, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.2 K 2024-12-03T14:14:23,037 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b598d2f5513e47488cf0a05b7ed37985, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733235257817 2024-12-03T14:14:23,037 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 040708b48bd044638a3d14fb896f3a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733235258391 2024-12-03T14:14:23,037 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3df6d8b1e648e58bb2cd270010a81d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:23,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742026_1202 (size=31801) 2024-12-03T14:14:23,062 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:23,063 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/58337d31871e46aaa35c67b7a291349c is 50, key is test_row_0/C:col10/1733235260587/Put/seqid=0 2024-12-03T14:14:23,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742027_1203 (size=12847) 2024-12-03T14:14:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T14:14:23,426 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:23,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T14:14:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T14:14:23,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:23,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:23,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:23,459 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2772b1405e5141a9859741e3f4465b34 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34 2024-12-03T14:14:23,465 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 2772b1405e5141a9859741e3f4465b34(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:23,465 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:23,465 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235263000; duration=0sec 2024-12-03T14:14:23,465 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:23,465 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:23,478 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/58337d31871e46aaa35c67b7a291349c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/58337d31871e46aaa35c67b7a291349c 2024-12-03T14:14:23,486 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into 58337d31871e46aaa35c67b7a291349c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:23,486 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:23,486 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235263000; duration=0sec 2024-12-03T14:14:23,487 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:23,487 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T14:14:23,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T14:14:23,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:23,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:23,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120304a81de0867948bfa6d8d86b05141710_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235261756/Put/seqid=0 2024-12-03T14:14:23,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742028_1204 (size=12454) 2024-12-03T14:14:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:23,602 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120304a81de0867948bfa6d8d86b05141710_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120304a81de0867948bfa6d8d86b05141710_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c8b896706b96428690a660ff200cc668, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:23,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c8b896706b96428690a660ff200cc668 is 175, key is test_row_0/A:col10/1733235261756/Put/seqid=0 2024-12-03T14:14:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742029_1205 (size=31255) 2024-12-03T14:14:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T14:14:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:23,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235323934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:23,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235323936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,018 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c8b896706b96428690a660ff200cc668 2024-12-03T14:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a00d51b63dc84c1891df55d00c04d9c0 is 50, key is test_row_0/B:col10/1733235261756/Put/seqid=0 2024-12-03T14:14:24,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742030_1206 (size=12301) 2024-12-03T14:14:24,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235324046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235324046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T14:14:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235324256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235324256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,475 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a00d51b63dc84c1891df55d00c04d9c0 2024-12-03T14:14:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/d16b380e90494e36b7e2717120d31774 is 50, key is test_row_0/C:col10/1733235261756/Put/seqid=0 2024-12-03T14:14:24,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742031_1207 (size=12301) 2024-12-03T14:14:24,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/d16b380e90494e36b7e2717120d31774 2024-12-03T14:14:24,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/c8b896706b96428690a660ff200cc668 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668 2024-12-03T14:14:24,499 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668, entries=150, sequenceid=285, filesize=30.5 K 2024-12-03T14:14:24,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a00d51b63dc84c1891df55d00c04d9c0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0 2024-12-03T14:14:24,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T14:14:24,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/d16b380e90494e36b7e2717120d31774 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774 2024-12-03T14:14:24,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774, entries=150, sequenceid=285, filesize=12.0 K 2024-12-03T14:14:24,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6bbe8ee77f3e0ec92707554117dec558 in 936ms, sequenceid=285, compaction requested=false 2024-12-03T14:14:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T14:14:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T14:14:24,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T14:14:24,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0900 sec 2024-12-03T14:14:24,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0950 sec 2024-12-03T14:14:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T14:14:24,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:24,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-03T14:14:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:24,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:14:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:24,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:24,572 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:24,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:24,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:24,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030f10c70e3e154426af3b94c6834f3cfa_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:24,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742032_1208 (size=14994) 2024-12-03T14:14:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235324615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235324618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:24,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:24,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235324726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235324726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:24,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:24,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:24,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:24,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235324936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235324936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:24,994 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:24,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030f10c70e3e154426af3b94c6834f3cfa_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f10c70e3e154426af3b94c6834f3cfa_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:24,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3921bb3a5599436590db881f7bc84204, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:25,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3921bb3a5599436590db881f7bc84204 is 175, key is test_row_0/A:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742033_1209 (size=39949) 2024-12-03T14:14:25,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3921bb3a5599436590db881f7bc84204 2024-12-03T14:14:25,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/702761fd662f475d9c79de2b6e7dfbda is 50, key is test_row_0/B:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:25,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742034_1210 (size=12301) 2024-12-03T14:14:25,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235325246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235325246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:25,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/702761fd662f475d9c79de2b6e7dfbda 2024-12-03T14:14:25,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0535572d7be24d149554b9b3abe7f58b is 50, key is test_row_0/C:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742035_1211 (size=12301) 2024-12-03T14:14:25,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235325756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235325756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:25,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:25,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0535572d7be24d149554b9b3abe7f58b 2024-12-03T14:14:25,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3921bb3a5599436590db881f7bc84204 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204 2024-12-03T14:14:25,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204, entries=200, sequenceid=299, filesize=39.0 K 2024-12-03T14:14:25,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/702761fd662f475d9c79de2b6e7dfbda as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda 2024-12-03T14:14:25,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T14:14:25,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/0535572d7be24d149554b9b3abe7f58b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b 2024-12-03T14:14:25,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T14:14:25,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6bbe8ee77f3e0ec92707554117dec558 in 1323ms, sequenceid=299, compaction requested=true 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:25,892 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:25,892 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:25,894 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,894 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,894 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=100.6 K 2024-12-03T14:14:25,894 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/e4455634825e4529947e2e84f07789ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.6 K 2024-12-03T14:14:25,894 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204] 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2772b1405e5141a9859741e3f4465b34, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:25,894 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e4455634825e4529947e2e84f07789ac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:25,895 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a00d51b63dc84c1891df55d00c04d9c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733235261754 2024-12-03T14:14:25,895 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b896706b96428690a660ff200cc668, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733235261754 2024-12-03T14:14:25,895 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3921bb3a5599436590db881f7bc84204, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:25,895 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 702761fd662f475d9c79de2b6e7dfbda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:25,902 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:25,903 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/1c98df801d584414b2e3397812a44e24 is 50, key is test_row_0/B:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:25,904 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:25,906 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f5d80afb720248cfa18d6f7c6ddadf3b_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:25,908 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f5d80afb720248cfa18d6f7c6ddadf3b_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:25,908 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f5d80afb720248cfa18d6f7c6ddadf3b_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742036_1212 (size=12949) 2024-12-03T14:14:25,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742037_1213 (size=4469) 2024-12-03T14:14:25,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:25,949 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120368e299c52e7845da99f2df73bcafa4c9_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235264613/Put/seqid=0 2024-12-03T14:14:25,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742038_1214 (size=12454) 2024-12-03T14:14:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:26,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235326036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235326036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235326036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235326146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235326146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235326146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,316 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/1c98df801d584414b2e3397812a44e24 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1c98df801d584414b2e3397812a44e24 2024-12-03T14:14:26,321 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into 1c98df801d584414b2e3397812a44e24(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:26,321 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:26,321 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235265892; duration=0sec 2024-12-03T14:14:26,321 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:26,321 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:26,321 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:26,322 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:26,322 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:26,323 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:26,323 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/58337d31871e46aaa35c67b7a291349c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.6 K 2024-12-03T14:14:26,323 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:26,323 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 58337d31871e46aaa35c67b7a291349c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235260566 2024-12-03T14:14:26,323 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/6b82d90ebdf44ffabf292e54a103c0b4 is 175, key is test_row_0/A:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:26,324 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d16b380e90494e36b7e2717120d31774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733235261754 2024-12-03T14:14:26,324 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0535572d7be24d149554b9b3abe7f58b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:26,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742039_1215 (size=31903) 2024-12-03T14:14:26,332 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/6b82d90ebdf44ffabf292e54a103c0b4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4 2024-12-03T14:14:26,334 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:26,334 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/a68a613bc43143a5b7ee7142bc15109b is 50, key is test_row_0/C:col10/1733235264569/Put/seqid=0 2024-12-03T14:14:26,340 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 6b82d90ebdf44ffabf292e54a103c0b4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:26,340 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:26,340 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235265892; duration=0sec 2024-12-03T14:14:26,340 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:26,340 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742040_1216 (size=12949) 2024-12-03T14:14:26,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235326356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235326356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235326357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,369 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120368e299c52e7845da99f2df73bcafa4c9_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120368e299c52e7845da99f2df73bcafa4c9_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e53cf5f6fd2e48769c5547de0660c3ba, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:26,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e53cf5f6fd2e48769c5547de0660c3ba is 175, key is test_row_0/A:col10/1733235264613/Put/seqid=0 2024-12-03T14:14:26,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742041_1217 (size=31255) 2024-12-03T14:14:26,378 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e53cf5f6fd2e48769c5547de0660c3ba 2024-12-03T14:14:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/4457652705f34a4a93487ed8d476fd65 is 50, key is test_row_0/B:col10/1733235264613/Put/seqid=0 2024-12-03T14:14:26,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742042_1218 (size=12301) 2024-12-03T14:14:26,396 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/4457652705f34a4a93487ed8d476fd65 2024-12-03T14:14:26,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5600550d20f247f08b0eb93d694542d5 is 50, key is test_row_0/C:col10/1733235264613/Put/seqid=0 2024-12-03T14:14:26,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742043_1219 (size=12301) 2024-12-03T14:14:26,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5600550d20f247f08b0eb93d694542d5 2024-12-03T14:14:26,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e53cf5f6fd2e48769c5547de0660c3ba as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba 2024-12-03T14:14:26,430 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba, entries=150, sequenceid=317, filesize=30.5 K 2024-12-03T14:14:26,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/4457652705f34a4a93487ed8d476fd65 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65 2024-12-03T14:14:26,436 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T14:14:26,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/5600550d20f247f08b0eb93d694542d5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5 2024-12-03T14:14:26,443 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T14:14:26,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6bbe8ee77f3e0ec92707554117dec558 in 495ms, sequenceid=317, compaction requested=false 2024-12-03T14:14:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:26,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-03T14:14:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-03T14:14:26,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T14:14:26,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-12-03T14:14:26,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8810 sec 2024-12-03T14:14:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:26,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:14:26,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:26,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:26,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c32e0c36901142088be78ad30e5b2a22_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:26,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742044_1220 (size=14994) 2024-12-03T14:14:26,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235326688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235326691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235326692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T14:14:26,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:26,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-03T14:14:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:26,719 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:26,719 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:26,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:26,754 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/a68a613bc43143a5b7ee7142bc15109b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a68a613bc43143a5b7ee7142bc15109b 2024-12-03T14:14:26,758 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into a68a613bc43143a5b7ee7142bc15109b(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:26,758 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:26,758 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235265892; duration=0sec 2024-12-03T14:14:26,759 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:26,759 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235326766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,769 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235326776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,779 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235326796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235326796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235326796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:26,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:26,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235327005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235327006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235327006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:27,079 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:27,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c32e0c36901142088be78ad30e5b2a22_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c32e0c36901142088be78ad30e5b2a22_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:27,083 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/bfb8c927f2c743af8086a03840d0c261, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:27,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/bfb8c927f2c743af8086a03840d0c261 is 175, key is test_row_0/A:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:27,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742045_1221 (size=39949) 2024-12-03T14:14:27,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235327316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235327316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235327316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:27,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/bfb8c927f2c743af8086a03840d0c261 2024-12-03T14:14:27,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/9ee3def0175e44a69c98bb1c69b9ba6a is 50, key is test_row_0/B:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:27,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742046_1222 (size=12301) 2024-12-03T14:14:27,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235327826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235327826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235327826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:27,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/9ee3def0175e44a69c98bb1c69b9ba6a 2024-12-03T14:14:27,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/955f3be829e9416db299c72549317607 is 50, key is test_row_0/C:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742047_1223 (size=12301) 2024-12-03T14:14:27,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/955f3be829e9416db299c72549317607 2024-12-03T14:14:27,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:27,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/bfb8c927f2c743af8086a03840d0c261 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261 2024-12-03T14:14:27,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261, entries=200, sequenceid=334, filesize=39.0 K 2024-12-03T14:14:27,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/9ee3def0175e44a69c98bb1c69b9ba6a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a 2024-12-03T14:14:27,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a, entries=150, sequenceid=334, filesize=12.0 K 2024-12-03T14:14:27,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/955f3be829e9416db299c72549317607 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607 2024-12-03T14:14:27,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607, entries=150, sequenceid=334, filesize=12.0 K 2024-12-03T14:14:27,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6bbe8ee77f3e0ec92707554117dec558 in 1295ms, sequenceid=334, compaction requested=true 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:27,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:27,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:27,963 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,963 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,963 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=100.7 K 2024-12-03T14:14:27,963 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1c98df801d584414b2e3397812a44e24, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.7 K 2024-12-03T14:14:27,963 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261] 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b82d90ebdf44ffabf292e54a103c0b4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:27,963 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c98df801d584414b2e3397812a44e24, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:27,964 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4457652705f34a4a93487ed8d476fd65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733235264610 2024-12-03T14:14:27,964 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53cf5f6fd2e48769c5547de0660c3ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733235264610 2024-12-03T14:14:27,964 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee3def0175e44a69c98bb1c69b9ba6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:27,964 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfb8c927f2c743af8086a03840d0c261, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:27,971 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:27,973 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:27,973 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c0325322540545cd85b00099f3de3cef_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:27,974 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/295245bca6604a2ba71dc758589c86cb is 50, key is test_row_0/B:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:27,975 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c0325322540545cd85b00099f3de3cef_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:27,975 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c0325322540545cd85b00099f3de3cef_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:27,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742048_1224 (size=13051) 2024-12-03T14:14:27,985 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/295245bca6604a2ba71dc758589c86cb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/295245bca6604a2ba71dc758589c86cb 2024-12-03T14:14:27,991 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into 295245bca6604a2ba71dc758589c86cb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:27,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:27,991 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235267962; duration=0sec 2024-12-03T14:14:27,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:27,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:27,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:27,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742049_1225 (size=4469) 2024-12-03T14:14:27,995 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:27,995 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:27,995 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:27,995 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a68a613bc43143a5b7ee7142bc15109b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.7 K 2024-12-03T14:14:27,995 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#186 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:27,996 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/cc7752a19c5949c3881529667aaed5c5 is 175, key is test_row_0/A:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:27,997 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a68a613bc43143a5b7ee7142bc15109b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733235263922 2024-12-03T14:14:27,998 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5600550d20f247f08b0eb93d694542d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733235264610 2024-12-03T14:14:27,998 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 955f3be829e9416db299c72549317607, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742050_1226 (size=32005) 2024-12-03T14:14:28,010 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:28,010 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/f3431dbfa2794f2d8baa63f699237ba1 is 50, key is test_row_0/C:col10/1733235266027/Put/seqid=0 2024-12-03T14:14:28,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742051_1227 (size=13051) 2024-12-03T14:14:28,026 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/f3431dbfa2794f2d8baa63f699237ba1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/f3431dbfa2794f2d8baa63f699237ba1 2024-12-03T14:14:28,033 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into f3431dbfa2794f2d8baa63f699237ba1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:28,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:28,033 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235267962; duration=0sec 2024-12-03T14:14:28,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:28,033 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:28,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:28,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030bb554dee989444f9de8fd3e3cd69e19_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235266691/Put/seqid=0 2024-12-03T14:14:28,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742052_1228 (size=12454) 2024-12-03T14:14:28,411 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/cc7752a19c5949c3881529667aaed5c5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5 2024-12-03T14:14:28,416 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into cc7752a19c5949c3881529667aaed5c5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:28,416 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:28,416 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235267962; duration=0sec 2024-12-03T14:14:28,416 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:28,416 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,512 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030bb554dee989444f9de8fd3e3cd69e19_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bb554dee989444f9de8fd3e3cd69e19_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e228d730e5ba497c9ed7497b1b223058, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e228d730e5ba497c9ed7497b1b223058 is 175, key is test_row_0/A:col10/1733235266691/Put/seqid=0 2024-12-03T14:14:28,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742053_1229 (size=31255) 2024-12-03T14:14:28,542 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e228d730e5ba497c9ed7497b1b223058 2024-12-03T14:14:28,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b8872e69c03c4d7dbd5941750e6a57d2 is 50, key is test_row_0/B:col10/1733235266691/Put/seqid=0 2024-12-03T14:14:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742054_1230 (size=12301) 2024-12-03T14:14:28,562 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b8872e69c03c4d7dbd5941750e6a57d2 2024-12-03T14:14:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/01c9c98f5f6549b48b4b4cc199475ace is 50, key is test_row_0/C:col10/1733235266691/Put/seqid=0 2024-12-03T14:14:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742055_1231 (size=12301) 2024-12-03T14:14:28,603 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/01c9c98f5f6549b48b4b4cc199475ace 2024-12-03T14:14:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e228d730e5ba497c9ed7497b1b223058 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058 2024-12-03T14:14:28,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058, entries=150, sequenceid=354, filesize=30.5 K 2024-12-03T14:14:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/b8872e69c03c4d7dbd5941750e6a57d2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2 2024-12-03T14:14:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,618 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T14:14:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/01c9c98f5f6549b48b4b4cc199475ace as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace 2024-12-03T14:14:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,625 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T14:14:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,626 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 6bbe8ee77f3e0ec92707554117dec558 in 532ms, sequenceid=354, compaction requested=false 2024-12-03T14:14:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-03T14:14:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-03T14:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T14:14:28,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9090 sec 2024-12-03T14:14:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9140 sec 2024-12-03T14:14:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:28,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:28,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:28,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203867e182da29a4d25a88da3d66bcf6106_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T14:14:28,867 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-03T14:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,870 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:28,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:28,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742056_1232 (size=22618) 2024-12-03T14:14:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235268842, max timestamp=1733235268928Num uniques 859; 2024-12-03T14:14:28,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:14:28,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235268842, max timestamp=1733235268928Num uniques 859; 2024-12-03T14:14:28,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:14:28,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235268842, max timestamp=1733235268928Num uniques 859; 2024-12-03T14:14:28,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:14:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235328933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235328933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235328942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235328944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235328947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:29,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:29,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235329056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235329056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235329056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235329056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235329076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:29,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235329266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235329266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235329267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235329267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235329286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,325 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:29,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203867e182da29a4d25a88da3d66bcf6106_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203867e182da29a4d25a88da3d66bcf6106_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:29,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4dac67951edc4c4ead2afb4bfebca3c1, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:29,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4dac67951edc4c4ead2afb4bfebca3c1 is 175, key is test_row_0/A:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742057_1233 (size=66023) 2024-12-03T14:14:29,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4dac67951edc4c4ead2afb4bfebca3c1 2024-12-03T14:14:29,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/08079e80bda1464bb2958495d12b3b91 is 50, key is test_row_0/B:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742058_1234 (size=12301) 2024-12-03T14:14:29,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235329576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235329576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235329576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235329576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235329597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:29,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:29,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/08079e80bda1464bb2958495d12b3b91 2024-12-03T14:14:29,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/475a5966a5924bb9aea38462f9f35ae4 is 50, key is test_row_0/C:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742059_1235 (size=12301) 2024-12-03T14:14:29,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/475a5966a5924bb9aea38462f9f35ae4 2024-12-03T14:14:29,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4dac67951edc4c4ead2afb4bfebca3c1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1 2024-12-03T14:14:29,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1, entries=350, sequenceid=365, filesize=64.5 K 2024-12-03T14:14:29,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/08079e80bda1464bb2958495d12b3b91 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91 2024-12-03T14:14:29,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T14:14:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/475a5966a5924bb9aea38462f9f35ae4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4 2024-12-03T14:14:29,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T14:14:29,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 6bbe8ee77f3e0ec92707554117dec558 in 940ms, sequenceid=365, compaction requested=true 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:29,782 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:29,782 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:29,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:29,783 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:29,783 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:29,783 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:29,783 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:29,783 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,783 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,783 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=126.3 K 2024-12-03T14:14:29,783 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/295245bca6604a2ba71dc758589c86cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.8 K 2024-12-03T14:14:29,783 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,783 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1] 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7752a19c5949c3881529667aaed5c5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 295245bca6604a2ba71dc758589c86cb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e228d730e5ba497c9ed7497b1b223058, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235266685 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b8872e69c03c4d7dbd5941750e6a57d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235266685 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dac67951edc4c4ead2afb4bfebca3c1, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268818 2024-12-03T14:14:29,784 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 08079e80bda1464bb2958495d12b3b91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268827 2024-12-03T14:14:29,789 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:29,791 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:29,791 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/acc77255f71348b2984261c7a1483e4e is 50, key is test_row_0/B:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,793 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412031cfe03066fab415fa55e093d32e86913_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:29,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412031cfe03066fab415fa55e093d32e86913_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:29,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031cfe03066fab415fa55e093d32e86913_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:29,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742060_1236 (size=13153) 2024-12-03T14:14:29,802 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/acc77255f71348b2984261c7a1483e4e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/acc77255f71348b2984261c7a1483e4e 2024-12-03T14:14:29,808 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into acc77255f71348b2984261c7a1483e4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:29,808 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:29,808 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235269782; duration=0sec 2024-12-03T14:14:29,808 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:29,808 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:29,808 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:29,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742061_1237 (size=4469) 2024-12-03T14:14:29,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:29,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:29,810 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,810 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/f3431dbfa2794f2d8baa63f699237ba1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.8 K 2024-12-03T14:14:29,811 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f3431dbfa2794f2d8baa63f699237ba1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733235266027 2024-12-03T14:14:29,811 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#195 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:29,812 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3d5542e978834761945ff4b8288074f6 is 175, key is test_row_0/A:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,812 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c9c98f5f6549b48b4b4cc199475ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733235266685 2024-12-03T14:14:29,815 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 475a5966a5924bb9aea38462f9f35ae4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268827 2024-12-03T14:14:29,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742062_1238 (size=32107) 2024-12-03T14:14:29,823 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/3d5542e978834761945ff4b8288074f6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6 2024-12-03T14:14:29,824 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:29,825 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/c57e198cb3f541688fb7858bcf34f374 is 50, key is test_row_0/C:col10/1733235268840/Put/seqid=0 2024-12-03T14:14:29,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742063_1239 (size=13153) 2024-12-03T14:14:29,830 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 3d5542e978834761945ff4b8288074f6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:29,830 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:29,830 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235269782; duration=0sec 2024-12-03T14:14:29,831 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:29,831 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:29,834 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/c57e198cb3f541688fb7858bcf34f374 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/c57e198cb3f541688fb7858bcf34f374 2024-12-03T14:14:29,839 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into c57e198cb3f541688fb7858bcf34f374(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:29,839 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:29,839 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235269782; duration=0sec 2024-12-03T14:14:29,839 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:29,839 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:29,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:29,844 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:29,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd23653b16c14d2ba1cb3db6c159773b_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235268940/Put/seqid=0 2024-12-03T14:14:29,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742064_1240 (size=12454) 2024-12-03T14:14:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:30,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:30,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235330089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235330092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235330093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235330097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235330106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235330206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235330206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235330206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235330206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:30,259 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cd23653b16c14d2ba1cb3db6c159773b_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd23653b16c14d2ba1cb3db6c159773b_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:30,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/95225eaeefcc4e00adfbad67c1889009, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:30,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/95225eaeefcc4e00adfbad67c1889009 is 175, key is test_row_0/A:col10/1733235268940/Put/seqid=0 2024-12-03T14:14:30,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742065_1241 (size=31255) 2024-12-03T14:14:30,265 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/95225eaeefcc4e00adfbad67c1889009 2024-12-03T14:14:30,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/19a70f7614364460bcca59e72c258b12 is 50, key is test_row_0/B:col10/1733235268940/Put/seqid=0 2024-12-03T14:14:30,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742066_1242 (size=12301) 2024-12-03T14:14:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235330416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235330416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235330416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235330416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,675 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/19a70f7614364460bcca59e72c258b12 2024-12-03T14:14:30,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/31bc51ef185f4758ae167083624c61a0 is 50, key is test_row_0/C:col10/1733235268940/Put/seqid=0 2024-12-03T14:14:30,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742067_1243 (size=12301) 2024-12-03T14:14:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235330726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235330727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235330727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235330727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:31,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/31bc51ef185f4758ae167083624c61a0 2024-12-03T14:14:31,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/95225eaeefcc4e00adfbad67c1889009 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009 2024-12-03T14:14:31,093 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009, entries=150, sequenceid=394, filesize=30.5 K 2024-12-03T14:14:31,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/19a70f7614364460bcca59e72c258b12 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12 2024-12-03T14:14:31,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T14:14:31,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/31bc51ef185f4758ae167083624c61a0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0 2024-12-03T14:14:31,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T14:14:31,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for 6bbe8ee77f3e0ec92707554117dec558 in 1259ms, sequenceid=394, compaction requested=false 2024-12-03T14:14:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:31,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-03T14:14:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-03T14:14:31,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T14:14:31,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-12-03T14:14:31,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2380 sec 2024-12-03T14:14:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:31,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:31,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031c74e4545ac64e95b0d9300cc4c13ca6_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:31,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742068_1244 (size=14994) 2024-12-03T14:14:31,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235271168, max timestamp=1733235271183Num uniques 859; 2024-12-03T14:14:31,185 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:31,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235271168, max timestamp=1733235271183Num uniques 859; 2024-12-03T14:14:31,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235271168, max timestamp=1733235271183Num uniques 859; 2024-12-03T14:14:31,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:31,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235331246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235331246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235331249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235331250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235331251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235331356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235331356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235331356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:31,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031c74e4545ac64e95b0d9300cc4c13ca6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031c74e4545ac64e95b0d9300cc4c13ca6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:31,538 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/00e48814eb084c8abd7d79ec3d70847a, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:31,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/00e48814eb084c8abd7d79ec3d70847a is 175, key is test_row_0/A:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:31,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742069_1245 (size=39949) 2024-12-03T14:14:31,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235331565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235331566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235331565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235331876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235331876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235331876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:31,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/00e48814eb084c8abd7d79ec3d70847a 2024-12-03T14:14:31,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/79aec705d6ec4aa4b43f47e566b6bd40 is 50, key is test_row_0/B:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:31,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742070_1246 (size=12301) 2024-12-03T14:14:32,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235332266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:32,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235332266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:32,268 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:32,268 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:32,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/79aec705d6ec4aa4b43f47e566b6bd40 2024-12-03T14:14:32,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/a3fb92b0d0234264945d5292b2fa9f99 is 50, key is test_row_0/C:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:32,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742071_1247 (size=12301) 2024-12-03T14:14:32,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/a3fb92b0d0234264945d5292b2fa9f99 2024-12-03T14:14:32,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/00e48814eb084c8abd7d79ec3d70847a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a 2024-12-03T14:14:32,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a, entries=200, sequenceid=405, filesize=39.0 K 2024-12-03T14:14:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/79aec705d6ec4aa4b43f47e566b6bd40 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40 2024-12-03T14:14:32,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T14:14:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/a3fb92b0d0234264945d5292b2fa9f99 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99 2024-12-03T14:14:32,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235332386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:32,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235332386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:32,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235332386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:32,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99, entries=150, sequenceid=405, filesize=12.0 K 2024-12-03T14:14:32,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6bbe8ee77f3e0ec92707554117dec558 in 1276ms, sequenceid=405, compaction requested=true 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:32,393 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:32,393 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:32,395 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:32,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:32,395 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/acc77255f71348b2984261c7a1483e4e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.9 K 2024-12-03T14:14:32,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=100.9 K 2024-12-03T14:14:32,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a] 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting acc77255f71348b2984261c7a1483e4e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268827 2024-12-03T14:14:32,395 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5542e978834761945ff4b8288074f6, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268827 2024-12-03T14:14:32,396 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a70f7614364460bcca59e72c258b12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235268928 2024-12-03T14:14:32,396 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95225eaeefcc4e00adfbad67c1889009, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235268928 2024-12-03T14:14:32,396 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 79aec705d6ec4aa4b43f47e566b6bd40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:32,396 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e48814eb084c8abd7d79ec3d70847a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:32,401 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:32,403 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:32,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/9fdbfeb13cc44355b8c2099f69f82eec is 50, key is test_row_0/B:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:32,404 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203be9bcdea4254487b80cbff6bb8bf73a3_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:32,405 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203be9bcdea4254487b80cbff6bb8bf73a3_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:32,405 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203be9bcdea4254487b80cbff6bb8bf73a3_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:32,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742072_1248 (size=13255) 2024-12-03T14:14:32,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742073_1249 (size=4469) 2024-12-03T14:14:32,813 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/9fdbfeb13cc44355b8c2099f69f82eec as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9fdbfeb13cc44355b8c2099f69f82eec 2024-12-03T14:14:32,818 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into 9fdbfeb13cc44355b8c2099f69f82eec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:32,818 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:32,818 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235272393; duration=0sec 2024-12-03T14:14:32,818 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:32,818 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:32,818 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:32,819 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:32,819 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:32,819 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:32,819 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/c57e198cb3f541688fb7858bcf34f374, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=36.9 K 2024-12-03T14:14:32,820 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c57e198cb3f541688fb7858bcf34f374, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235268827 2024-12-03T14:14:32,820 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 31bc51ef185f4758ae167083624c61a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235268928 2024-12-03T14:14:32,820 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#204 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:32,820 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a3fb92b0d0234264945d5292b2fa9f99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:32,821 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e5cf68fe2ee44bf581f84dfe1f0fde41 is 175, key is test_row_0/A:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:32,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742074_1250 (size=32209) 2024-12-03T14:14:32,831 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:32,832 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6862a84023a1441598c265aedaad4cc1 is 50, key is test_row_0/C:col10/1733235270092/Put/seqid=0 2024-12-03T14:14:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742075_1251 (size=13255) 2024-12-03T14:14:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T14:14:33,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:33,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-03T14:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:33,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:33,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:33,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:33,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T14:14:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:33,181 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T14:14:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fad941825a8243ea85d31d2f59ed17a2_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235271249/Put/seqid=0 2024-12-03T14:14:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742076_1252 (size=12454) 2024-12-03T14:14:33,231 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/e5cf68fe2ee44bf581f84dfe1f0fde41 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41 2024-12-03T14:14:33,235 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into e5cf68fe2ee44bf581f84dfe1f0fde41(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:33,235 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:33,236 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235272393; duration=0sec 2024-12-03T14:14:33,236 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:33,236 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:33,243 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6862a84023a1441598c265aedaad4cc1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6862a84023a1441598c265aedaad4cc1 2024-12-03T14:14:33,248 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into 6862a84023a1441598c265aedaad4cc1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:33,248 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:33,248 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235272393; duration=0sec 2024-12-03T14:14:33,248 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:33,248 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. as already flushing 2024-12-03T14:14:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235333406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235333407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235333407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235333516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235333516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235333516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:33,595 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fad941825a8243ea85d31d2f59ed17a2_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fad941825a8243ea85d31d2f59ed17a2_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:33,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/1b6a61aa608d4ea3b002edac69e66287, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:33,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/1b6a61aa608d4ea3b002edac69e66287 is 175, key is test_row_0/A:col10/1733235271249/Put/seqid=0 2024-12-03T14:14:33,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742077_1253 (size=31255) 2024-12-03T14:14:33,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/1b6a61aa608d4ea3b002edac69e66287 2024-12-03T14:14:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/7d6eb57a32b540cdbb40d30b24fdc1b2 is 50, key is test_row_0/B:col10/1733235271249/Put/seqid=0 2024-12-03T14:14:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742078_1254 (size=12301) 2024-12-03T14:14:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:33,698 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-12-03T14:14:33,698 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:33,698 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,698 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:33,700 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-12-03T14:14:33,700 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:33,700 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,700 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:33,701 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-12-03T14:14:33,701 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:33,701 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,701 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:33,701 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-12-03T14:14:33,701 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:33,701 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,701 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:33,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235333727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235333727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235333727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,022 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/7d6eb57a32b540cdbb40d30b24fdc1b2 2024-12-03T14:14:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59996 deadline: 1733235334036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59978 deadline: 1733235334036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:34,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:34,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/09298f3d43ea48c9b64072c4df48b98e is 50, key is test_row_0/C:col10/1733235271249/Put/seqid=0 2024-12-03T14:14:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60030 deadline: 1733235334037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742079_1255 (size=12301) 2024-12-03T14:14:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60024 deadline: 1733235334287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:34,288 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59976 deadline: 1733235334297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:34,298 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 6bbe8ee77f3e0ec92707554117dec558 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:14:34,443 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/09298f3d43ea48c9b64072c4df48b98e 2024-12-03T14:14:34,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/1b6a61aa608d4ea3b002edac69e66287 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287 2024-12-03T14:14:34,462 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287, entries=150, sequenceid=442, filesize=30.5 K 2024-12-03T14:14:34,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/7d6eb57a32b540cdbb40d30b24fdc1b2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2 2024-12-03T14:14:34,468 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2, entries=150, sequenceid=442, filesize=12.0 K 2024-12-03T14:14:34,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/09298f3d43ea48c9b64072c4df48b98e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e 2024-12-03T14:14:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e, entries=150, sequenceid=442, filesize=12.0 K 2024-12-03T14:14:34,474 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6bbe8ee77f3e0ec92707554117dec558 in 1293ms, sequenceid=442, compaction requested=false 2024-12-03T14:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-03T14:14:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-03T14:14:34,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T14:14:34,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-12-03T14:14:34,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4510 sec 2024-12-03T14:14:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:34,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:14:34,561 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-12-03T14:14:34,561 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-12-03T14:14:34,561 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:34,561 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-12-03T14:14:34,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:34,561 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:34,561 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,561 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,562 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:34,562 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,562 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:34,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:34,562 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:34,562 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:34,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:34,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:34,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:34,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036777316cfdba4c6bbf59eab3b19f1d87_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_0/A:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742080_1256 (size=12454) 2024-12-03T14:14:34,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:34,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036777316cfdba4c6bbf59eab3b19f1d87_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036777316cfdba4c6bbf59eab3b19f1d87_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:34,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4a9f812c3804447599731798968a1fe4, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:34,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4a9f812c3804447599731798968a1fe4 is 175, key is test_row_0/A:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:34,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742081_1257 (size=31255) 2024-12-03T14:14:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T14:14:35,187 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:35,395 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4a9f812c3804447599731798968a1fe4 2024-12-03T14:14:35,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a383bfb646c040aa9d2590ccaedd7e68 is 50, key is test_row_0/B:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:35,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742082_1258 (size=12301) 2024-12-03T14:14:35,458 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:14:35,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a383bfb646c040aa9d2590ccaedd7e68 2024-12-03T14:14:35,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6e62a547f3de44f8aecb683e5692c296 is 50, key is test_row_0/C:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:35,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742083_1259 (size=12301) 2024-12-03T14:14:36,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6e62a547f3de44f8aecb683e5692c296 2024-12-03T14:14:36,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/4a9f812c3804447599731798968a1fe4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4 2024-12-03T14:14:36,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4, entries=150, sequenceid=457, filesize=30.5 K 2024-12-03T14:14:36,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/a383bfb646c040aa9d2590ccaedd7e68 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68 2024-12-03T14:14:36,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68, entries=150, sequenceid=457, filesize=12.0 K 2024-12-03T14:14:36,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/6e62a547f3de44f8aecb683e5692c296 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296 2024-12-03T14:14:36,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296, entries=150, sequenceid=457, filesize=12.0 K 2024-12-03T14:14:36,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 6bbe8ee77f3e0ec92707554117dec558 in 1696ms, sequenceid=457, compaction requested=true 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:36,256 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bbe8ee77f3e0ec92707554117dec558:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:36,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:36,256 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:36,257 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:36,257 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:36,257 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/A is initiating minor compaction (all files) 2024-12-03T14:14:36,257 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/B is initiating minor compaction (all files) 2024-12-03T14:14:36,257 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/A in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:36,258 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/B in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:36,258 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=92.5 K 2024-12-03T14:14:36,258 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9fdbfeb13cc44355b8c2099f69f82eec, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=37.0 K 2024-12-03T14:14:36,258 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4] 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fdbfeb13cc44355b8c2099f69f82eec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cf68fe2ee44bf581f84dfe1f0fde41, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6eb57a32b540cdbb40d30b24fdc1b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235271225 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6a61aa608d4ea3b002edac69e66287, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235271225 2024-12-03T14:14:36,258 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a383bfb646c040aa9d2590ccaedd7e68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733235273397 2024-12-03T14:14:36,259 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a9f812c3804447599731798968a1fe4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733235273397 2024-12-03T14:14:36,267 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:36,268 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/69d52eccf0e0492f85a21e8b3f00bede is 50, key is test_row_0/B:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:36,269 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:36,272 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203e1869174dde6496ea6122d4f949bec58_6bbe8ee77f3e0ec92707554117dec558 store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:36,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742084_1260 (size=13357) 2024-12-03T14:14:36,283 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203e1869174dde6496ea6122d4f949bec58_6bbe8ee77f3e0ec92707554117dec558, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:36,283 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e1869174dde6496ea6122d4f949bec58_6bbe8ee77f3e0ec92707554117dec558 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:36,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742085_1261 (size=4469) 2024-12-03T14:14:36,687 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/69d52eccf0e0492f85a21e8b3f00bede as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/69d52eccf0e0492f85a21e8b3f00bede 2024-12-03T14:14:36,691 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/B of 6bbe8ee77f3e0ec92707554117dec558 into 69d52eccf0e0492f85a21e8b3f00bede(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:36,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:36,691 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/B, priority=13, startTime=1733235276256; duration=0sec 2024-12-03T14:14:36,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:36,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:B 2024-12-03T14:14:36,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:36,692 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#A#compaction#214 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:36,692 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:36,692 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 6bbe8ee77f3e0ec92707554117dec558/C is initiating minor compaction (all files) 2024-12-03T14:14:36,692 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bbe8ee77f3e0ec92707554117dec558/C in TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:36,692 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6862a84023a1441598c265aedaad4cc1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp, totalSize=37.0 K 2024-12-03T14:14:36,693 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/5171b2dbd9154ce4995f33e571bdb75e is 175, key is test_row_0/A:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:36,693 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6862a84023a1441598c265aedaad4cc1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733235270090 2024-12-03T14:14:36,693 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 09298f3d43ea48c9b64072c4df48b98e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1733235271225 2024-12-03T14:14:36,693 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e62a547f3de44f8aecb683e5692c296, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1733235273397 2024-12-03T14:14:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742086_1262 (size=32311) 2024-12-03T14:14:36,700 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bbe8ee77f3e0ec92707554117dec558#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:36,701 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/9334cf2b7cbc49c3afe745da655481cb is 50, key is test_row_0/C:col10/1733235274547/Put/seqid=0 2024-12-03T14:14:36,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742087_1263 (size=13357) 2024-12-03T14:14:37,109 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/5171b2dbd9154ce4995f33e571bdb75e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/5171b2dbd9154ce4995f33e571bdb75e 2024-12-03T14:14:37,113 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/9334cf2b7cbc49c3afe745da655481cb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9334cf2b7cbc49c3afe745da655481cb 2024-12-03T14:14:37,116 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/A of 6bbe8ee77f3e0ec92707554117dec558 into 5171b2dbd9154ce4995f33e571bdb75e(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:37,116 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:37,116 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/A, priority=13, startTime=1733235276256; duration=0sec 2024-12-03T14:14:37,117 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:37,117 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:A 2024-12-03T14:14:37,120 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bbe8ee77f3e0ec92707554117dec558/C of 6bbe8ee77f3e0ec92707554117dec558 into 9334cf2b7cbc49c3afe745da655481cb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:37,120 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bbe8ee77f3e0ec92707554117dec558: 2024-12-03T14:14:37,120 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., storeName=6bbe8ee77f3e0ec92707554117dec558/C, priority=13, startTime=1733235276256; duration=0sec 2024-12-03T14:14:37,120 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:37,120 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bbe8ee77f3e0ec92707554117dec558:C 2024-12-03T14:14:38,331 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-12-03T14:14:38,331 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:38,332 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,332 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:38,338 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-12-03T14:14:38,338 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:14:38,338 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,338 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4096 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4098 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1828 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5484 rows 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1839 2024-12-03T14:14:38,338 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5517 rows 2024-12-03T14:14:38,339 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:14:38,339 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:14:38,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:38,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:14:38,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:14:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:38,345 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235278344"}]},"ts":"1733235278344"} 2024-12-03T14:14:38,347 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:14:38,347 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:14:38,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:14:38,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, UNASSIGN}] 2024-12-03T14:14:38,352 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, UNASSIGN 2024-12-03T14:14:38,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:38,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, UNASSIGN because future has completed 2024-12-03T14:14:38,354 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:14:38,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:38,507 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:38,507 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:14:38,507 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 6bbe8ee77f3e0ec92707554117dec558, disabling compactions & flushes 2024-12-03T14:14:38,507 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:38,507 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:38,507 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. after waiting 0 ms 2024-12-03T14:14:38,507 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:38,508 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 6bbe8ee77f3e0ec92707554117dec558 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=A 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=B 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6bbe8ee77f3e0ec92707554117dec558, store=C 2024-12-03T14:14:38,508 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:38,520 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036295fde748134efda71d685c7495a044_6bbe8ee77f3e0ec92707554117dec558 is 50, key is test_row_1/A:col10/1733235278336/Put/seqid=0 2024-12-03T14:14:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742088_1264 (size=9914) 2024-12-03T14:14:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:38,925 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:38,933 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036295fde748134efda71d685c7495a044_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036295fde748134efda71d685c7495a044_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:38,935 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2a1b8b34c0f94cbb87447c7021c6ebbc, store: [table=TestAcidGuarantees family=A region=6bbe8ee77f3e0ec92707554117dec558] 2024-12-03T14:14:38,936 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2a1b8b34c0f94cbb87447c7021c6ebbc is 175, key is test_row_1/A:col10/1733235278336/Put/seqid=0 2024-12-03T14:14:38,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742089_1265 (size=22561) 2024-12-03T14:14:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:39,342 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2a1b8b34c0f94cbb87447c7021c6ebbc 2024-12-03T14:14:39,355 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/1435a5de4fd8409f876cf850af3fd3f3 is 50, key is test_row_1/B:col10/1733235278336/Put/seqid=0 2024-12-03T14:14:39,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742090_1266 (size=9857) 2024-12-03T14:14:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:39,761 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/1435a5de4fd8409f876cf850af3fd3f3 2024-12-03T14:14:39,776 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/9cff87af0f834b75964e6c00598588ec is 50, key is test_row_1/C:col10/1733235278336/Put/seqid=0 2024-12-03T14:14:39,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742091_1267 (size=9857) 2024-12-03T14:14:40,183 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/9cff87af0f834b75964e6c00598588ec 2024-12-03T14:14:40,194 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/A/2a1b8b34c0f94cbb87447c7021c6ebbc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2a1b8b34c0f94cbb87447c7021c6ebbc 2024-12-03T14:14:40,199 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2a1b8b34c0f94cbb87447c7021c6ebbc, entries=100, sequenceid=465, filesize=22.0 K 2024-12-03T14:14:40,200 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/B/1435a5de4fd8409f876cf850af3fd3f3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1435a5de4fd8409f876cf850af3fd3f3 2024-12-03T14:14:40,203 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1435a5de4fd8409f876cf850af3fd3f3, entries=100, sequenceid=465, filesize=9.6 K 2024-12-03T14:14:40,204 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/.tmp/C/9cff87af0f834b75964e6c00598588ec as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9cff87af0f834b75964e6c00598588ec 2024-12-03T14:14:40,208 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9cff87af0f834b75964e6c00598588ec, entries=100, sequenceid=465, filesize=9.6 K 2024-12-03T14:14:40,208 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6bbe8ee77f3e0ec92707554117dec558 in 1701ms, sequenceid=465, compaction requested=false 2024-12-03T14:14:40,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4] to archive 2024-12-03T14:14:40,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:40,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/9dac9a60fff6434ea6ab65f987ec1962 2024-12-03T14:14:40,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e8a29798c5df4959965a2281df158167 2024-12-03T14:14:40,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8b19907aabc34ab59cee114fe8e4dcb9 2024-12-03T14:14:40,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8643b1e22077498da41581e22c6079d5 2024-12-03T14:14:40,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/681ff7859bc44ed19d4993f7ace89664 2024-12-03T14:14:40,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/344ca28e69554cd08837f1ec66d99930 2024-12-03T14:14:40,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74f940b9c12240d8a707ed38cb34ee57 2024-12-03T14:14:40,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c1f86b62968e4b1b9faac3120de109b5 2024-12-03T14:14:40,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/74c7fa6ec4b941cb8931ada6529f2a17 2024-12-03T14:14:40,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/759526c5614e4521ac2fcd82dc80f19f 2024-12-03T14:14:40,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/d43fd3b2704a4b6098910918d652cd27 2024-12-03T14:14:40,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e1d1625a2cf3499f9e0447b74b46a38b 2024-12-03T14:14:40,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/f9678e485026408fbbf9cb89408e179a 2024-12-03T14:14:40,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/51616e51c4154fe8a0fdbe79428270e6 2024-12-03T14:14:40,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/8063d63a6afa4ae4b62ddfc3336cc788 2024-12-03T14:14:40,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/df5613b7b3524808896ab1f7daae64b7 2024-12-03T14:14:40,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2772b1405e5141a9859741e3f4465b34 2024-12-03T14:14:40,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/c8b896706b96428690a660ff200cc668 2024-12-03T14:14:40,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3921bb3a5599436590db881f7bc84204 2024-12-03T14:14:40,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/6b82d90ebdf44ffabf292e54a103c0b4 2024-12-03T14:14:40,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e53cf5f6fd2e48769c5547de0660c3ba 2024-12-03T14:14:40,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/bfb8c927f2c743af8086a03840d0c261 2024-12-03T14:14:40,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/cc7752a19c5949c3881529667aaed5c5 2024-12-03T14:14:40,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e228d730e5ba497c9ed7497b1b223058 2024-12-03T14:14:40,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4dac67951edc4c4ead2afb4bfebca3c1 2024-12-03T14:14:40,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/3d5542e978834761945ff4b8288074f6 2024-12-03T14:14:40,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/95225eaeefcc4e00adfbad67c1889009 2024-12-03T14:14:40,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/00e48814eb084c8abd7d79ec3d70847a 2024-12-03T14:14:40,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/e5cf68fe2ee44bf581f84dfe1f0fde41 2024-12-03T14:14:40,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/1b6a61aa608d4ea3b002edac69e66287 2024-12-03T14:14:40,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/4a9f812c3804447599731798968a1fe4 2024-12-03T14:14:40,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19951f9708554d188d6a2f9b6c8b6d92, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/eb99349f8f054447bf9c249cd7c77b91, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/d596b097d1ac4f358b983fb70793b6f4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/f29d3894b1ef499c91ad6a8cced0985d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/e4455634825e4529947e2e84f07789ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1c98df801d584414b2e3397812a44e24, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/295245bca6604a2ba71dc758589c86cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/acc77255f71348b2984261c7a1483e4e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9fdbfeb13cc44355b8c2099f69f82eec, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68] to archive 2024-12-03T14:14:40,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:40,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a71ce853d9b14f799680cca413a3350a 2024-12-03T14:14:40,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/90045bd539584e1b931e378efb3c96e1 2024-12-03T14:14:40,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19951f9708554d188d6a2f9b6c8b6d92 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19951f9708554d188d6a2f9b6c8b6d92 2024-12-03T14:14:40,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/972b7099c0f94289a39024794295fdc0 2024-12-03T14:14:40,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/063d7464b30a4eee96e28b852b5c3710 2024-12-03T14:14:40,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/eb99349f8f054447bf9c249cd7c77b91 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/eb99349f8f054447bf9c249cd7c77b91 2024-12-03T14:14:40,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a08ea8cdf84b4ac1881775ac3428cd2c 2024-12-03T14:14:40,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/85bedd175107455d895776c3ad7ef3f1 2024-12-03T14:14:40,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b4bc439c968e434c8ca2dbdc23539d08 2024-12-03T14:14:40,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/d596b097d1ac4f358b983fb70793b6f4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/d596b097d1ac4f358b983fb70793b6f4 2024-12-03T14:14:40,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/0502dacfe83740cabb501495de77cd77 2024-12-03T14:14:40,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/3c26d91de02a48e0a25c6c9ed9d98a7b 2024-12-03T14:14:40,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/f29d3894b1ef499c91ad6a8cced0985d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/f29d3894b1ef499c91ad6a8cced0985d 2024-12-03T14:14:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/bfa7706f9dfa4794bc78585995f996f0 2024-12-03T14:14:40,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/28a2c2928bed440cb9f6aecb51ac4ba3 2024-12-03T14:14:40,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/e4455634825e4529947e2e84f07789ac to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/e4455634825e4529947e2e84f07789ac 2024-12-03T14:14:40,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/48d74826bc754fca97dd1e208d856c07 2024-12-03T14:14:40,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a00d51b63dc84c1891df55d00c04d9c0 2024-12-03T14:14:40,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1c98df801d584414b2e3397812a44e24 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1c98df801d584414b2e3397812a44e24 2024-12-03T14:14:40,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/702761fd662f475d9c79de2b6e7dfbda 2024-12-03T14:14:40,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/4457652705f34a4a93487ed8d476fd65 2024-12-03T14:14:40,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/295245bca6604a2ba71dc758589c86cb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/295245bca6604a2ba71dc758589c86cb 2024-12-03T14:14:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9ee3def0175e44a69c98bb1c69b9ba6a 2024-12-03T14:14:40,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/b8872e69c03c4d7dbd5941750e6a57d2 2024-12-03T14:14:40,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/acc77255f71348b2984261c7a1483e4e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/acc77255f71348b2984261c7a1483e4e 2024-12-03T14:14:40,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/08079e80bda1464bb2958495d12b3b91 2024-12-03T14:14:40,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/19a70f7614364460bcca59e72c258b12 2024-12-03T14:14:40,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9fdbfeb13cc44355b8c2099f69f82eec to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/9fdbfeb13cc44355b8c2099f69f82eec 2024-12-03T14:14:40,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/79aec705d6ec4aa4b43f47e566b6bd40 2024-12-03T14:14:40,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/7d6eb57a32b540cdbb40d30b24fdc1b2 2024-12-03T14:14:40,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/a383bfb646c040aa9d2590ccaedd7e68 2024-12-03T14:14:40,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/417c20daabd64673928975b11ffb1a4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/72202d608d434cdcac6d851524e83b0d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b5c393ae6edb43fd9d797f5e6f5eec34, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b598d2f5513e47488cf0a05b7ed37985, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/58337d31871e46aaa35c67b7a291349c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a68a613bc43143a5b7ee7142bc15109b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/f3431dbfa2794f2d8baa63f699237ba1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/c57e198cb3f541688fb7858bcf34f374, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6862a84023a1441598c265aedaad4cc1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296] to archive 2024-12-03T14:14:40,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:14:40,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/ad954e4b0c5d41938d400cfa2d72afa6 2024-12-03T14:14:40,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/054f15262b5d480390bcbafc96dc1339 2024-12-03T14:14:40,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/417c20daabd64673928975b11ffb1a4b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/417c20daabd64673928975b11ffb1a4b 2024-12-03T14:14:40,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/28839f4c8c354af78978dd164b3cc416 2024-12-03T14:14:40,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/197b74e584384ae9ad0b965a6fa80f35 2024-12-03T14:14:40,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/72202d608d434cdcac6d851524e83b0d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/72202d608d434cdcac6d851524e83b0d 2024-12-03T14:14:40,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6215838bb01b4c9cb731a64ef6a3e7c1 2024-12-03T14:14:40,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5035ccdbcb68442987252490d29d043d 2024-12-03T14:14:40,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/979ee46ae35c4bcc88755375f8bc5f83 2024-12-03T14:14:40,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b5c393ae6edb43fd9d797f5e6f5eec34 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b5c393ae6edb43fd9d797f5e6f5eec34 2024-12-03T14:14:40,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0daeb97241ef4d0eb98944e095d5f2ea 2024-12-03T14:14:40,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/52c60350e20a4c328b0a54219c03161e 2024-12-03T14:14:40,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b598d2f5513e47488cf0a05b7ed37985 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/b598d2f5513e47488cf0a05b7ed37985 2024-12-03T14:14:40,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/2fc0b94764054e5fbf2b31d7d198f33c 2024-12-03T14:14:40,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/040708b48bd044638a3d14fb896f3a82 2024-12-03T14:14:40,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/58337d31871e46aaa35c67b7a291349c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/58337d31871e46aaa35c67b7a291349c 2024-12-03T14:14:40,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/1f3df6d8b1e648e58bb2cd270010a81d 2024-12-03T14:14:40,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/d16b380e90494e36b7e2717120d31774 2024-12-03T14:14:40,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a68a613bc43143a5b7ee7142bc15109b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a68a613bc43143a5b7ee7142bc15109b 2024-12-03T14:14:40,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/0535572d7be24d149554b9b3abe7f58b 2024-12-03T14:14:40,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/5600550d20f247f08b0eb93d694542d5 2024-12-03T14:14:40,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/f3431dbfa2794f2d8baa63f699237ba1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/f3431dbfa2794f2d8baa63f699237ba1 2024-12-03T14:14:40,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/955f3be829e9416db299c72549317607 2024-12-03T14:14:40,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/01c9c98f5f6549b48b4b4cc199475ace 2024-12-03T14:14:40,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/c57e198cb3f541688fb7858bcf34f374 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/c57e198cb3f541688fb7858bcf34f374 2024-12-03T14:14:40,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/475a5966a5924bb9aea38462f9f35ae4 2024-12-03T14:14:40,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/31bc51ef185f4758ae167083624c61a0 2024-12-03T14:14:40,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6862a84023a1441598c265aedaad4cc1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6862a84023a1441598c265aedaad4cc1 2024-12-03T14:14:40,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/a3fb92b0d0234264945d5292b2fa9f99 2024-12-03T14:14:40,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/09298f3d43ea48c9b64072c4df48b98e 2024-12-03T14:14:40,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/6e62a547f3de44f8aecb683e5692c296 2024-12-03T14:14:40,306 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=4 2024-12-03T14:14:40,307 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. 2024-12-03T14:14:40,307 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 6bbe8ee77f3e0ec92707554117dec558: Waiting for close lock at 1733235278507Running coprocessor pre-close hooks at 1733235278507Disabling compacts and flushes for region at 1733235278507Disabling writes for close at 1733235278507Obtaining lock to block concurrent updates at 1733235278508 (+1 ms)Preparing flush snapshotting stores in 6bbe8ee77f3e0ec92707554117dec558 at 1733235278508Finished memstore snapshotting TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733235278508Flushing stores of TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558. at 1733235278509 (+1 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/A: creating writer at 1733235278510 (+1 ms)Flushing mob file 6bbe8ee77f3e0ec92707554117dec558/A: appending metadata at 1733235278520 (+10 ms)Flushing mob file 6bbe8ee77f3e0ec92707554117dec558/A: closing flushed file at 1733235278520Flushing 6bbe8ee77f3e0ec92707554117dec558/A: appending metadata at 1733235278935 (+415 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/A: closing flushed file at 1733235278936 (+1 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/B: creating writer at 1733235279351 (+415 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/B: appending metadata at 1733235279355 (+4 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/B: closing flushed file at 1733235279355Flushing 6bbe8ee77f3e0ec92707554117dec558/C: creating writer at 1733235279766 (+411 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/C: appending metadata at 1733235279775 (+9 ms)Flushing 6bbe8ee77f3e0ec92707554117dec558/C: closing flushed file at 1733235279776 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d9f1102: reopening flushed file at 1733235280193 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ef3db34: reopening flushed file at 1733235280199 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40c7e8f3: reopening flushed file at 1733235280203 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 6bbe8ee77f3e0ec92707554117dec558 in 1701ms, sequenceid=465, compaction requested=false at 1733235280208 (+5 ms)Writing region close event to WAL at 1733235280303 (+95 ms)Running coprocessor post-close hooks at 1733235280307 (+4 ms)Closed at 1733235280307 2024-12-03T14:14:40,309 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,309 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=6bbe8ee77f3e0ec92707554117dec558, regionState=CLOSED 2024-12-03T14:14:40,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:40,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T14:14:40,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 6bbe8ee77f3e0ec92707554117dec558, server=3920e6a7c3b5,42177,1733235217010 in 1.9570 sec 2024-12-03T14:14:40,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-03T14:14:40,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bbe8ee77f3e0ec92707554117dec558, UNASSIGN in 1.9620 sec 2024-12-03T14:14:40,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T14:14:40,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9660 sec 2024-12-03T14:14:40,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235280317"}]},"ts":"1733235280317"} 2024-12-03T14:14:40,319 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:14:40,319 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:14:40,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9790 sec 2024-12-03T14:14:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T14:14:40,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:40,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:14:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,490 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T14:14:40,492 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,497 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,502 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits] 2024-12-03T14:14:40,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2a1b8b34c0f94cbb87447c7021c6ebbc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/2a1b8b34c0f94cbb87447c7021c6ebbc 2024-12-03T14:14:40,507 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/5171b2dbd9154ce4995f33e571bdb75e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/A/5171b2dbd9154ce4995f33e571bdb75e 2024-12-03T14:14:40,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1435a5de4fd8409f876cf850af3fd3f3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/1435a5de4fd8409f876cf850af3fd3f3 2024-12-03T14:14:40,510 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/69d52eccf0e0492f85a21e8b3f00bede to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/B/69d52eccf0e0492f85a21e8b3f00bede 2024-12-03T14:14:40,512 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9334cf2b7cbc49c3afe745da655481cb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9334cf2b7cbc49c3afe745da655481cb 2024-12-03T14:14:40,513 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9cff87af0f834b75964e6c00598588ec to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/C/9cff87af0f834b75964e6c00598588ec 2024-12-03T14:14:40,515 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits/468.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558/recovered.edits/468.seqid 2024-12-03T14:14:40,515 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,515 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:14:40,516 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:14:40,516 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T14:14:40,520 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120304a81de0867948bfa6d8d86b05141710_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120304a81de0867948bfa6d8d86b05141710_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,521 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bb554dee989444f9de8fd3e3cd69e19_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bb554dee989444f9de8fd3e3cd69e19_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,522 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e48c22cb3b04df5a37d53660bb44cd0_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030e48c22cb3b04df5a37d53660bb44cd0_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,523 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f10c70e3e154426af3b94c6834f3cfa_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f10c70e3e154426af3b94c6834f3cfa_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,524 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031c74e4545ac64e95b0d9300cc4c13ca6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031c74e4545ac64e95b0d9300cc4c13ca6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,525 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120327f96c572ce14b82b6ed5bf2685bdfd6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120327f96c572ce14b82b6ed5bf2685bdfd6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,526 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033b9b4485e91641d0b1c1fcacb7cc10a7_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033b9b4485e91641d0b1c1fcacb7cc10a7_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c88119717e342ea9319336d5db2e7b0_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035c88119717e342ea9319336d5db2e7b0_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,527 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036295fde748134efda71d685c7495a044_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036295fde748134efda71d685c7495a044_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,528 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036719ac4311c94703b6164baf17c390b5_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036719ac4311c94703b6164baf17c390b5_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,529 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036777316cfdba4c6bbf59eab3b19f1d87_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036777316cfdba4c6bbf59eab3b19f1d87_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,530 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120368e299c52e7845da99f2df73bcafa4c9_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120368e299c52e7845da99f2df73bcafa4c9_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,531 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378cfafdefb0d41ed81a3e7df6365c616_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120378cfafdefb0d41ed81a3e7df6365c616_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,532 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037e916bc37a4b420b911760f274711d6d_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037e916bc37a4b420b911760f274711d6d_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,533 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203867e182da29a4d25a88da3d66bcf6106_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203867e182da29a4d25a88da3d66bcf6106_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,534 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cfd12ef41ed4bd78fa234b5c4beb70e_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cfd12ef41ed4bd78fa234b5c4beb70e_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,535 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a0435aa16db94b10bb52112063d09824_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a0435aa16db94b10bb52112063d09824_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,536 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c32e0c36901142088be78ad30e5b2a22_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c32e0c36901142088be78ad30e5b2a22_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,537 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd23653b16c14d2ba1cb3db6c159773b_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cd23653b16c14d2ba1cb3db6c159773b_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,538 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cdfd34fae63c408cbd0b77b91462bdab_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cdfd34fae63c408cbd0b77b91462bdab_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,539 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4ea1aa092eb48b1b2b511b05854e301_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4ea1aa092eb48b1b2b511b05854e301_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,540 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3c20a59fe6748c3a28892c97005a9c6_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f3c20a59fe6748c3a28892c97005a9c6_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,541 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fad941825a8243ea85d31d2f59ed17a2_6bbe8ee77f3e0ec92707554117dec558 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fad941825a8243ea85d31d2f59ed17a2_6bbe8ee77f3e0ec92707554117dec558 2024-12-03T14:14:40,541 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:14:40,543 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,546 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:14:40,548 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:14:40,549 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,549 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:14:40,550 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235280549"}]},"ts":"9223372036854775807"} 2024-12-03T14:14:40,552 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:14:40,552 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6bbe8ee77f3e0ec92707554117dec558, NAME => 'TestAcidGuarantees,,1733235250506.6bbe8ee77f3e0ec92707554117dec558.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:14:40,552 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:14:40,552 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235280552"}]},"ts":"9223372036854775807"} 2024-12-03T14:14:40,555 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:14:40,556 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-12-03T14:14:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T14:14:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:14:40,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:40,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1738973269_22 at /127.0.0.1:34490 [Waiting for operation #780] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1738973269_22 at /127.0.0.1:49966 [Waiting for operation #743] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_362070758_22 at /127.0.0.1:45128 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_362070758_22 at /127.0.0.1:45136 [Waiting for operation #239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 287) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10088 (was 10295) 2024-12-03T14:14:40,614 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=10087 2024-12-03T14:14:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:14:40,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:14:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:14:40,618 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:14:40,618 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:40,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-03T14:14:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:40,619 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:14:40,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742092_1268 (size=1061) 2024-12-03T14:14:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:41,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:14:41,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742093_1269 (size=53) 2024-12-03T14:14:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:41,438 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:14:41,438 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d37984c6fe08d63ee0981270943fab76, disabling compactions & flushes 2024-12-03T14:14:41,438 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,439 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,439 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. after waiting 0 ms 2024-12-03T14:14:41,439 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,439 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,439 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d37984c6fe08d63ee0981270943fab76: Waiting for close lock at 1733235281438Disabling compacts and flushes for region at 1733235281438Disabling writes for close at 1733235281439 (+1 ms)Writing region close event to WAL at 1733235281439Closed at 1733235281439 2024-12-03T14:14:41,442 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:14:41,442 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235281442"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235281442"}]},"ts":"1733235281442"} 2024-12-03T14:14:41,446 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:14:41,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:14:41,448 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235281447"}]},"ts":"1733235281447"} 2024-12-03T14:14:41,450 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:14:41,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, ASSIGN}] 2024-12-03T14:14:41,452 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, ASSIGN 2024-12-03T14:14:41,452 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:14:41,604 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=d37984c6fe08d63ee0981270943fab76, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:41,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, ASSIGN because future has completed 2024-12-03T14:14:41,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:14:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:41,770 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,770 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:14:41,771 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,771 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:14:41,771 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,771 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,774 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,777 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:41,777 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d37984c6fe08d63ee0981270943fab76 columnFamilyName A 2024-12-03T14:14:41,778 DEBUG [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:41,779 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(327): Store=d37984c6fe08d63ee0981270943fab76/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:41,779 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,780 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:41,780 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d37984c6fe08d63ee0981270943fab76 columnFamilyName B 2024-12-03T14:14:41,781 DEBUG [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:41,781 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(327): Store=d37984c6fe08d63ee0981270943fab76/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:41,781 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,782 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:14:41,783 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d37984c6fe08d63ee0981270943fab76 columnFamilyName C 2024-12-03T14:14:41,783 DEBUG [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:14:41,783 INFO [StoreOpener-d37984c6fe08d63ee0981270943fab76-1 {}] regionserver.HStore(327): Store=d37984c6fe08d63ee0981270943fab76/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:14:41,783 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,783 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,784 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,784 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,785 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,785 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,786 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:14:41,787 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,788 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:14:41,789 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened d37984c6fe08d63ee0981270943fab76; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70958167, jitterRate=0.057359084486961365}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:14:41,789 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:41,790 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for d37984c6fe08d63ee0981270943fab76: Running coprocessor pre-open hook at 1733235281771Writing region info on filesystem at 1733235281771Initializing all the Stores at 1733235281773 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235281773Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235281774 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235281774Cleaning up temporary data from old regions at 1733235281785 (+11 ms)Running coprocessor post-open hooks at 1733235281789 (+4 ms)Region opened successfully at 1733235281790 (+1 ms) 2024-12-03T14:14:41,790 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., pid=65, masterSystemTime=1733235281763 2024-12-03T14:14:41,792 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,792 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:41,792 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=d37984c6fe08d63ee0981270943fab76, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:41,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:14:41,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T14:14:41,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 in 185 msec 2024-12-03T14:14:41,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-03T14:14:41,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, ASSIGN in 346 msec 2024-12-03T14:14:41,798 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:14:41,798 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235281798"}]},"ts":"1733235281798"} 2024-12-03T14:14:41,800 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:14:41,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:14:41,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-12-03T14:14:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T14:14:42,778 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:42,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc05918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119e95ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,787 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,790 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:14:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:14:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14004fa8 2024-12-03T14:14:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:14:42,792 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:14:42,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,794 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1619f295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,796 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2423e35f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,798 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edec135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574d91df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,800 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79976e4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50288fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8b0451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a305a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c24601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dda3ace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8afb9a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103ab93d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,811 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,811 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,812 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d5fea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,813 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d52f596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e132854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bcc2916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,817 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13805f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a9a4e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:14:42,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:14:42,820 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:14:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:14:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:14:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f77168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:14:42,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:14:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:14:42,821 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:14:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2904f764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:14:42,822 DEBUG [Thread-1233 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1246 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1244 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,822 DEBUG [Thread-1237 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,823 DEBUG [Thread-1239 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,823 DEBUG [Thread-1235 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,824 DEBUG [Thread-1250 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,824 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,824 DEBUG [Thread-1250 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,824 DEBUG [Thread-1241 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:14:42,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T14:14:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:14:42,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,827 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:42,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,828 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:42,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:14:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:14:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:42,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:42,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:42,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/bff88f4f3bfc42a19f680a39279e9c40 is 50, key is test_row_0/A:col10/1733235282858/Put/seqid=0 2024-12-03T14:14:42,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235282865, max timestamp=1733235282880Num uniques 859; 2024-12-03T14:14:42,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235282865, max timestamp=1733235282880Num uniques 859; 2024-12-03T14:14:42,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:42,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:42,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235282865, max timestamp=1733235282880Num uniques 859; 2024-12-03T14:14:42,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:42,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742094_1270 (size=12001) 2024-12-03T14:14:42,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235342899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235342900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235342900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:42,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235342901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235342901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:42,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:42,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:42,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235343006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235343016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235343016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235343016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235343017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235343216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235343225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235343226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235343226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235343226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/bff88f4f3bfc42a19f680a39279e9c40 2024-12-03T14:14:43,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/7a97e0514ecf44179fc1e98f3d3228ed is 50, key is test_row_0/B:col10/1733235282858/Put/seqid=0 2024-12-03T14:14:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742095_1271 (size=12001) 2024-12-03T14:14:43,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/7a97e0514ecf44179fc1e98f3d3228ed 2024-12-03T14:14:43,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a141cb84e7da4f778c2413d9e2a0dcbc is 50, key is test_row_0/C:col10/1733235282858/Put/seqid=0 2024-12-03T14:14:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742096_1272 (size=12001) 2024-12-03T14:14:43,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235343526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235343536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235343536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235343536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235343536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:43,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:43,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a141cb84e7da4f778c2413d9e2a0dcbc 2024-12-03T14:14:43,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/bff88f4f3bfc42a19f680a39279e9c40 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40 2024-12-03T14:14:43,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:14:43,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/7a97e0514ecf44179fc1e98f3d3228ed as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed 2024-12-03T14:14:43,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:14:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a141cb84e7da4f778c2413d9e2a0dcbc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc 2024-12-03T14:14:43,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T14:14:43,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d37984c6fe08d63ee0981270943fab76 in 922ms, sequenceid=12, compaction requested=false 2024-12-03T14:14:43,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:43,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T14:14:43,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:43,906 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:43,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/38c1a358accf4f178949a765c53a1c76 is 50, key is test_row_0/A:col10/1733235282899/Put/seqid=0 2024-12-03T14:14:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742097_1273 (size=12001) 2024-12-03T14:14:43,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:44,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:44,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235344049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235344050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235344050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235344050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235344051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235344166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235344166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235344166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235344166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235344166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,338 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/38c1a358accf4f178949a765c53a1c76 2024-12-03T14:14:44,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/fc195c78c9f34629bf557144ccbe58e6 is 50, key is test_row_0/B:col10/1733235282899/Put/seqid=0 2024-12-03T14:14:44,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742098_1274 (size=12001) 2024-12-03T14:14:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235344376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235344376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235344376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235344376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235344376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235344686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235344686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235344686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235344686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235344686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:44,748 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/fc195c78c9f34629bf557144ccbe58e6 2024-12-03T14:14:44,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/f8b310d0270f49da956e289c7d8dfe25 is 50, key is test_row_0/C:col10/1733235282899/Put/seqid=0 2024-12-03T14:14:44,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742099_1275 (size=12001) 2024-12-03T14:14:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:45,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235345196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:45,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235345196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235345197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235345197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235345197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:45,211 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/f8b310d0270f49da956e289c7d8dfe25 2024-12-03T14:14:45,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/38c1a358accf4f178949a765c53a1c76 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76 2024-12-03T14:14:45,218 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T14:14:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/fc195c78c9f34629bf557144ccbe58e6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6 2024-12-03T14:14:45,223 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T14:14:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/f8b310d0270f49da956e289c7d8dfe25 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25 2024-12-03T14:14:45,228 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T14:14:45,229 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for d37984c6fe08d63ee0981270943fab76 in 1324ms, sequenceid=49, compaction requested=false 2024-12-03T14:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:45,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T14:14:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T14:14:45,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T14:14:45,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4010 sec 2024-12-03T14:14:45,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.4080 sec 2024-12-03T14:14:45,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:14:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:46,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:46,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:46,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/95af8d647d5a4c538c0f4544b213c651 is 50, key is test_row_0/A:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742100_1276 (size=16681) 2024-12-03T14:14:46,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/95af8d647d5a4c538c0f4544b213c651 2024-12-03T14:14:46,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f94bac6b08174e3d8478fb7700c55ec5 is 50, key is test_row_0/B:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742101_1277 (size=12001) 2024-12-03T14:14:46,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235346226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235346229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235346229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235346230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235346231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235346336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235346336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235346336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235346336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235346336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235346545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235346546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235346546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235346546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235346546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f94bac6b08174e3d8478fb7700c55ec5 2024-12-03T14:14:46,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a454f82bbb9545cc9adc957807730c6a is 50, key is test_row_0/C:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742102_1278 (size=12001) 2024-12-03T14:14:46,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a454f82bbb9545cc9adc957807730c6a 2024-12-03T14:14:46,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/95af8d647d5a4c538c0f4544b213c651 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651 2024-12-03T14:14:46,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651, entries=250, sequenceid=62, filesize=16.3 K 2024-12-03T14:14:46,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f94bac6b08174e3d8478fb7700c55ec5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5 2024-12-03T14:14:46,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T14:14:46,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a454f82bbb9545cc9adc957807730c6a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a 2024-12-03T14:14:46,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T14:14:46,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d37984c6fe08d63ee0981270943fab76 in 447ms, sequenceid=62, compaction requested=true 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:46,655 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:46,655 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:46,656 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:46,656 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:46,656 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:14:46,656 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:14:46,656 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:46,656 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:46,656 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=39.7 K 2024-12-03T14:14:46,656 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.2 K 2024-12-03T14:14:46,656 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff88f4f3bfc42a19f680a39279e9c40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235282830 2024-12-03T14:14:46,657 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a97e0514ecf44179fc1e98f3d3228ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235282830 2024-12-03T14:14:46,657 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38c1a358accf4f178949a765c53a1c76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235282899 2024-12-03T14:14:46,657 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting fc195c78c9f34629bf557144ccbe58e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235282899 2024-12-03T14:14:46,657 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95af8d647d5a4c538c0f4544b213c651, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:46,657 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f94bac6b08174e3d8478fb7700c55ec5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:46,672 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:46,673 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/2fd351e72a174df68965c39ec7099d43 is 50, key is test_row_0/B:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,676 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:46,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742103_1279 (size=12104) 2024-12-03T14:14:46,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/320c28d9b18145d281cfb348de681dfc is 50, key is test_row_0/A:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,682 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/2fd351e72a174df68965c39ec7099d43 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2fd351e72a174df68965c39ec7099d43 2024-12-03T14:14:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742104_1280 (size=12104) 2024-12-03T14:14:46,688 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into 2fd351e72a174df68965c39ec7099d43(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:46,688 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:46,688 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=13, startTime=1733235286655; duration=0sec 2024-12-03T14:14:46,688 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:46,688 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:14:46,688 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:46,690 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:46,690 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:14:46,690 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:46,690 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.2 K 2024-12-03T14:14:46,690 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a141cb84e7da4f778c2413d9e2a0dcbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733235282830 2024-12-03T14:14:46,691 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/320c28d9b18145d281cfb348de681dfc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/320c28d9b18145d281cfb348de681dfc 2024-12-03T14:14:46,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b310d0270f49da956e289c7d8dfe25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733235282899 2024-12-03T14:14:46,691 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a454f82bbb9545cc9adc957807730c6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:46,697 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into 320c28d9b18145d281cfb348de681dfc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:46,697 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:46,697 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=13, startTime=1733235286655; duration=0sec 2024-12-03T14:14:46,697 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:46,697 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:14:46,702 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:46,703 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/7b35d9dd4a504ff08d6c40bf790fa561 is 50, key is test_row_0/C:col10/1733235284048/Put/seqid=0 2024-12-03T14:14:46,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742105_1281 (size=12104) 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235286224, max timestamp=1733235286856Num uniques 859; 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235286224, max timestamp=1733235286856Num uniques 859; 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235286224, max timestamp=1733235286856Num uniques 859; 2024-12-03T14:14:46,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:46,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:46,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/51052213e589440dba033f4e2dc9bbbd is 50, key is test_row_0/A:col10/1733235286859/Put/seqid=0 2024-12-03T14:14:46,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742106_1282 (size=12001) 2024-12-03T14:14:46,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235286892, max timestamp=1733235286902Num uniques 859; 2024-12-03T14:14:46,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:14:46,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235286892, max timestamp=1733235286902Num uniques 859; 2024-12-03T14:14:46,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:14:46,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235286897, max timestamp=1733235286902Num uniques 859; 2024-12-03T14:14:46,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:14:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235346917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235346918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235346919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235346920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235346919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:46,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T14:14:46,996 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:46,996 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T14:14:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:46,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:46,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:46,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235347026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235347036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235347036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235347036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235347037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:47,112 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/7b35d9dd4a504ff08d6c40bf790fa561 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7b35d9dd4a504ff08d6c40bf790fa561 2024-12-03T14:14:47,116 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into 7b35d9dd4a504ff08d6c40bf790fa561(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:47,116 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:47,116 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=13, startTime=1733235286655; duration=0sec 2024-12-03T14:14:47,116 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:47,117 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:14:47,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235347236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235347246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235347246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235347247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235347247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/51052213e589440dba033f4e2dc9bbbd 2024-12-03T14:14:47,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/e0572505dda04865ba7a103685a1cf8e is 50, key is test_row_0/B:col10/1733235286859/Put/seqid=0 2024-12-03T14:14:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742107_1283 (size=12001) 2024-12-03T14:14:47,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:47,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235347546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235347557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:47,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:47,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/e0572505dda04865ba7a103685a1cf8e 2024-12-03T14:14:47,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/ac9c37a3db4a48d9bdc95d495d23c92d is 50, key is test_row_0/C:col10/1733235286859/Put/seqid=0 2024-12-03T14:14:47,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742108_1284 (size=12001) 2024-12-03T14:14:47,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:47,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:48,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235348056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:48,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:48,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235348065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235348066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235348066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235348066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:48,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/ac9c37a3db4a48d9bdc95d495d23c92d 2024-12-03T14:14:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/51052213e589440dba033f4e2dc9bbbd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd 2024-12-03T14:14:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd, entries=150, sequenceid=89, filesize=11.7 K 2024-12-03T14:14:48,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/e0572505dda04865ba7a103685a1cf8e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e 2024-12-03T14:14:48,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e, entries=150, sequenceid=89, filesize=11.7 K 2024-12-03T14:14:48,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/ac9c37a3db4a48d9bdc95d495d23c92d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d 2024-12-03T14:14:48,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d, entries=150, sequenceid=89, filesize=11.7 K 2024-12-03T14:14:48,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=136.42 KB/139690 for d37984c6fe08d63ee0981270943fab76 in 1249ms, sequenceid=89, compaction requested=false 2024-12-03T14:14:48,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:48,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T14:14:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:48,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-03T14:14:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:48,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:48,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/41184cffc2f14823ba013d8022f0e305 is 50, key is test_row_0/A:col10/1733235286919/Put/seqid=0 2024-12-03T14:14:48,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742109_1285 (size=12001) 2024-12-03T14:14:48,628 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/41184cffc2f14823ba013d8022f0e305 2024-12-03T14:14:48,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 is 50, key is test_row_0/B:col10/1733235286919/Put/seqid=0 2024-12-03T14:14:48,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742110_1286 (size=12001) 2024-12-03T14:14:49,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 2024-12-03T14:14:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/c6fcf035c0a04717a87a302421b58b8e is 50, key is test_row_0/C:col10/1733235286919/Put/seqid=0 2024-12-03T14:14:49,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742111_1287 (size=12001) 2024-12-03T14:14:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:49,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235349102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235349105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235349106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235349107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235349107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235349216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235349216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235349216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235349216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235349216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235349426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235349426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235349426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235349426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235349426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,453 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/c6fcf035c0a04717a87a302421b58b8e 2024-12-03T14:14:49,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/41184cffc2f14823ba013d8022f0e305 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305 2024-12-03T14:14:49,460 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305, entries=150, sequenceid=126, filesize=11.7 K 2024-12-03T14:14:49,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 2024-12-03T14:14:49,464 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04, entries=150, sequenceid=126, filesize=11.7 K 2024-12-03T14:14:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/c6fcf035c0a04717a87a302421b58b8e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e 2024-12-03T14:14:49,468 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e, entries=150, sequenceid=126, filesize=11.7 K 2024-12-03T14:14:49,469 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=53.67 KB/54960 for d37984c6fe08d63ee0981270943fab76 in 1249ms, sequenceid=126, compaction requested=true 2024-12-03T14:14:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:49,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T14:14:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T14:14:49,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T14:14:49,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4710 sec 2024-12-03T14:14:49,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.4770 sec 2024-12-03T14:14:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:49,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:49,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:49,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/c8587ac0aaaa465e8f8d747aa726a40b is 50, key is test_row_0/A:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:49,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742112_1288 (size=12151) 2024-12-03T14:14:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235349790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235289748, max timestamp=1733235289794Num uniques 859; 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235289748, max timestamp=1733235289794Num uniques 859; 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:49,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235349791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235289748, max timestamp=1733235289794Num uniques 859; 2024-12-03T14:14:49,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:14:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235349816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235349817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235349820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235349905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235349906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235349926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235349926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235349926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235350115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235350116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235350136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235350136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235350136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/c8587ac0aaaa465e8f8d747aa726a40b 2024-12-03T14:14:50,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/61045bae5608472bbe6610036291be52 is 50, key is test_row_0/B:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742113_1289 (size=12151) 2024-12-03T14:14:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235350426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235350426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235350446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235350446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235350446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/61045bae5608472bbe6610036291be52 2024-12-03T14:14:50,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/64148c97b5764bad849e57aa69e04c6c is 50, key is test_row_0/C:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:50,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742114_1290 (size=12151) 2024-12-03T14:14:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235350936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235350936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235350955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235350956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235350956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:50,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:51,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/64148c97b5764bad849e57aa69e04c6c 2024-12-03T14:14:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/c8587ac0aaaa465e8f8d747aa726a40b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b 2024-12-03T14:14:51,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:14:51,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/61045bae5608472bbe6610036291be52 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52 2024-12-03T14:14:51,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:14:51,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/64148c97b5764bad849e57aa69e04c6c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c 2024-12-03T14:14:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:14:51,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for d37984c6fe08d63ee0981270943fab76 in 1284ms, sequenceid=138, compaction requested=true 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:51,021 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:51,021 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:51,022 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:51,022 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:51,022 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:14:51,022 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:14:51,022 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:51,022 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:51,022 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2fd351e72a174df68965c39ec7099d43, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=47.1 K 2024-12-03T14:14:51,022 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/320c28d9b18145d281cfb348de681dfc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=47.1 K 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd351e72a174df68965c39ec7099d43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320c28d9b18145d281cfb348de681dfc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51052213e589440dba033f4e2dc9bbbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733235286230 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e0572505dda04865ba7a103685a1cf8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733235286230 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41184cffc2f14823ba013d8022f0e305, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733235286917 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6c3cdbfd1848cd9b0643ca3ec24f04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733235286917 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8587ac0aaaa465e8f8d747aa726a40b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:51,023 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 61045bae5608472bbe6610036291be52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:51,029 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:51,030 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/5f240c908fdd4f2c8a7cad02b1279173 is 50, key is test_row_0/A:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:51,030 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:51,030 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/387efaa120884dd599d892c5abe4ff89 is 50, key is test_row_0/B:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742115_1291 (size=12391) 2024-12-03T14:14:51,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742116_1292 (size=12391) 2024-12-03T14:14:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T14:14:51,167 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:51,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T14:14:51,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:51,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:51,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:51,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T14:14:51,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:51,322 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:14:51,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/f9d271691b3e406e90c09b3273736603 is 50, key is test_row_0/A:col10/1733235289816/Put/seqid=0 2024-12-03T14:14:51,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742117_1293 (size=12151) 2024-12-03T14:14:51,437 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/5f240c908fdd4f2c8a7cad02b1279173 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/5f240c908fdd4f2c8a7cad02b1279173 2024-12-03T14:14:51,442 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into 5f240c908fdd4f2c8a7cad02b1279173(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:51,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:51,442 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=12, startTime=1733235291021; duration=0sec 2024-12-03T14:14:51,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:51,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:14:51,442 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:51,442 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/387efaa120884dd599d892c5abe4ff89 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/387efaa120884dd599d892c5abe4ff89 2024-12-03T14:14:51,443 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:51,443 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:14:51,443 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:51,443 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7b35d9dd4a504ff08d6c40bf790fa561, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=47.1 K 2024-12-03T14:14:51,444 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b35d9dd4a504ff08d6c40bf790fa561, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733235284048 2024-12-03T14:14:51,444 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac9c37a3db4a48d9bdc95d495d23c92d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733235286230 2024-12-03T14:14:51,444 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6fcf035c0a04717a87a302421b58b8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733235286917 2024-12-03T14:14:51,445 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64148c97b5764bad849e57aa69e04c6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:51,447 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into 387efaa120884dd599d892c5abe4ff89(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:51,447 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:51,447 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=12, startTime=1733235291021; duration=0sec 2024-12-03T14:14:51,447 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:51,447 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:14:51,453 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:51,453 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/d642bcc424014305913d2afe96324674 is 50, key is test_row_0/C:col10/1733235289106/Put/seqid=0 2024-12-03T14:14:51,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742118_1294 (size=12391) 2024-12-03T14:14:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:51,730 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/f9d271691b3e406e90c09b3273736603 2024-12-03T14:14:51,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/3a4dadddbcfe45719eca7cc6d563759c is 50, key is test_row_0/B:col10/1733235289816/Put/seqid=0 2024-12-03T14:14:51,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742119_1295 (size=12151) 2024-12-03T14:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:51,861 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/d642bcc424014305913d2afe96324674 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/d642bcc424014305913d2afe96324674 2024-12-03T14:14:51,866 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into d642bcc424014305913d2afe96324674(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:51,866 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:51,866 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=12, startTime=1733235291021; duration=0sec 2024-12-03T14:14:51,866 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:51,866 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:14:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:51,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235351965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:51,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235351966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:51,968 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:51,969 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235351967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:51,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235351968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:51,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235351976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:51,979 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:51,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235352076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235352076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,141 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/3a4dadddbcfe45719eca7cc6d563759c 2024-12-03T14:14:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/7e65157aebdb4012aa5ad4fc4519b17d is 50, key is test_row_0/C:col10/1733235289816/Put/seqid=0 2024-12-03T14:14:52,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742120_1296 (size=12151) 2024-12-03T14:14:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235352286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235352286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:52,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/7e65157aebdb4012aa5ad4fc4519b17d 2024-12-03T14:14:52,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/f9d271691b3e406e90c09b3273736603 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603 2024-12-03T14:14:52,561 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T14:14:52,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/3a4dadddbcfe45719eca7cc6d563759c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c 2024-12-03T14:14:52,565 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T14:14:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/7e65157aebdb4012aa5ad4fc4519b17d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d 2024-12-03T14:14:52,569 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T14:14:52,570 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for d37984c6fe08d63ee0981270943fab76 in 1248ms, sequenceid=172, compaction requested=false 2024-12-03T14:14:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T14:14:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T14:14:52,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T14:14:52,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-12-03T14:14:52,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4060 sec 2024-12-03T14:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:52,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:52,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b60f046045074fa3800c331687795f26 is 50, key is test_row_0/A:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742121_1297 (size=16931) 2024-12-03T14:14:52,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b60f046045074fa3800c331687795f26 2024-12-03T14:14:52,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/0fec4cf9bca14e95b475c760b348502d is 50, key is test_row_0/B:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:52,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742122_1298 (size=12151) 2024-12-03T14:14:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235352625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235352626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235352736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235352736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235352946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235352946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:52,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/0fec4cf9bca14e95b475c760b348502d 2024-12-03T14:14:53,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a2712ad021244b88b91dd4bab855d521 is 50, key is test_row_0/C:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:53,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742123_1299 (size=12151) 2024-12-03T14:14:53,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235353256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235353256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T14:14:53,316 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:53,316 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T14:14:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:53,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:53,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:53,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:53,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a2712ad021244b88b91dd4bab855d521 2024-12-03T14:14:53,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b60f046045074fa3800c331687795f26 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26 2024-12-03T14:14:53,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26, entries=250, sequenceid=189, filesize=16.5 K 2024-12-03T14:14:53,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/0fec4cf9bca14e95b475c760b348502d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d 2024-12-03T14:14:53,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d, entries=150, sequenceid=189, filesize=11.9 K 2024-12-03T14:14:53,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a2712ad021244b88b91dd4bab855d521 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521 2024-12-03T14:14:53,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521, entries=150, sequenceid=189, filesize=11.9 K 2024-12-03T14:14:53,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d37984c6fe08d63ee0981270943fab76 in 850ms, sequenceid=189, compaction requested=true 2024-12-03T14:14:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:53,449 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:53,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:53,450 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:53,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:53,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:53,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:53,450 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:14:53,451 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:53,451 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/387efaa120884dd599d892c5abe4ff89, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.8 K 2024-12-03T14:14:53,451 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:53,451 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/5f240c908fdd4f2c8a7cad02b1279173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=40.5 K 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 387efaa120884dd599d892c5abe4ff89, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f240c908fdd4f2c8a7cad02b1279173, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:53,451 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4dadddbcfe45719eca7cc6d563759c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235289813 2024-12-03T14:14:53,452 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d271691b3e406e90c09b3273736603, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235289813 2024-12-03T14:14:53,452 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fec4cf9bca14e95b475c760b348502d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:53,452 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b60f046045074fa3800c331687795f26, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:53,459 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:53,460 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/9a175a911c3a4db79a197cfc55f757af is 50, key is test_row_0/B:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:53,460 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:53,460 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/ae0469b47d90422d9f4a8515f4734e8e is 50, key is test_row_0/A:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:53,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742124_1300 (size=12493) 2024-12-03T14:14:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742125_1301 (size=12493) 2024-12-03T14:14:53,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:53,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:53,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/a3e940c827d84dcb8794ce9ebfdf9eea is 50, key is test_row_0/A:col10/1733235292615/Put/seqid=0 2024-12-03T14:14:53,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742126_1302 (size=12151) 2024-12-03T14:14:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:53,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235353792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235353792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,870 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/9a175a911c3a4db79a197cfc55f757af as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/9a175a911c3a4db79a197cfc55f757af 2024-12-03T14:14:53,874 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/ae0469b47d90422d9f4a8515f4734e8e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/ae0469b47d90422d9f4a8515f4734e8e 2024-12-03T14:14:53,874 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into 9a175a911c3a4db79a197cfc55f757af(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:53,874 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:53,874 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=13, startTime=1733235293449; duration=0sec 2024-12-03T14:14:53,874 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:53,874 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:14:53,874 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:53,875 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:53,875 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:14:53,875 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:53,875 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/d642bcc424014305913d2afe96324674, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.8 K 2024-12-03T14:14:53,875 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d642bcc424014305913d2afe96324674, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235289079 2024-12-03T14:14:53,876 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e65157aebdb4012aa5ad4fc4519b17d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733235289813 2024-12-03T14:14:53,876 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a2712ad021244b88b91dd4bab855d521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:53,878 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into ae0469b47d90422d9f4a8515f4734e8e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:53,878 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:53,878 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=13, startTime=1733235293449; duration=0sec 2024-12-03T14:14:53,878 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:53,878 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:14:53,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/a3e940c827d84dcb8794ce9ebfdf9eea 2024-12-03T14:14:53,884 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:53,885 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/5b9087b964fa4386853660cb2d30e799 is 50, key is test_row_0/C:col10/1733235292597/Put/seqid=0 2024-12-03T14:14:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/5bd81e1b229848f68adbbfec659f243f is 50, key is test_row_0/B:col10/1733235292615/Put/seqid=0 2024-12-03T14:14:53,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742127_1303 (size=12493) 2024-12-03T14:14:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742128_1304 (size=12151) 2024-12-03T14:14:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235353906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235353906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235353976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,977 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:53,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235353986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:53,989 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235354006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,007 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235354115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235354116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,292 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/5bd81e1b229848f68adbbfec659f243f 2024-12-03T14:14:54,297 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/5b9087b964fa4386853660cb2d30e799 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/5b9087b964fa4386853660cb2d30e799 2024-12-03T14:14:54,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0c8539d0639b4b4fbcfecad070b37784 is 50, key is test_row_0/C:col10/1733235292615/Put/seqid=0 2024-12-03T14:14:54,304 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into 5b9087b964fa4386853660cb2d30e799(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:54,304 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:54,304 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=13, startTime=1733235293450; duration=0sec 2024-12-03T14:14:54,304 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:54,304 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:14:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742129_1305 (size=12151) 2024-12-03T14:14:54,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235354426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235354426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:54,717 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0c8539d0639b4b4fbcfecad070b37784 2024-12-03T14:14:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/a3e940c827d84dcb8794ce9ebfdf9eea as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea 2024-12-03T14:14:54,730 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:14:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/5bd81e1b229848f68adbbfec659f243f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f 2024-12-03T14:14:54,734 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:14:54,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0c8539d0639b4b4fbcfecad070b37784 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784 2024-12-03T14:14:54,738 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T14:14:54,739 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d37984c6fe08d63ee0981270943fab76 in 1268ms, sequenceid=207, compaction requested=false 2024-12-03T14:14:54,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:54,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:54,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T14:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T14:14:54,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T14:14:54,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4230 sec 2024-12-03T14:14:54,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4260 sec 2024-12-03T14:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:54,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:54,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:54,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e52f938a88174796b29b7f30e754d695 is 50, key is test_row_0/A:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:54,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742130_1306 (size=14541) 2024-12-03T14:14:54,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e52f938a88174796b29b7f30e754d695 2024-12-03T14:14:54,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f964e747c0284759a42165099249b165 is 50, key is test_row_0/B:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:54,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742131_1307 (size=12151) 2024-12-03T14:14:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235354967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235354968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235355076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235355076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235355285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235355285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f964e747c0284759a42165099249b165 2024-12-03T14:14:55,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/bb81b63275ad4a70aaeb9b7ad195dacc is 50, key is test_row_0/C:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:55,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742132_1308 (size=12151) 2024-12-03T14:14:55,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/bb81b63275ad4a70aaeb9b7ad195dacc 2024-12-03T14:14:55,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e52f938a88174796b29b7f30e754d695 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695 2024-12-03T14:14:55,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695, entries=200, sequenceid=227, filesize=14.2 K 2024-12-03T14:14:55,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/f964e747c0284759a42165099249b165 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165 2024-12-03T14:14:55,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T14:14:55,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/bb81b63275ad4a70aaeb9b7ad195dacc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc 2024-12-03T14:14:55,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc, entries=150, sequenceid=227, filesize=11.9 K 2024-12-03T14:14:55,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d37984c6fe08d63ee0981270943fab76 in 449ms, sequenceid=227, compaction requested=true 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:55,387 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:55,387 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:55,388 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:55,388 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:55,388 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:14:55,388 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:14:55,388 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,388 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,388 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/9a175a911c3a4db79a197cfc55f757af, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.9 K 2024-12-03T14:14:55,388 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/ae0469b47d90422d9f4a8515f4734e8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=38.3 K 2024-12-03T14:14:55,388 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a175a911c3a4db79a197cfc55f757af, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:55,389 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0469b47d90422d9f4a8515f4734e8e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:55,389 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd81e1b229848f68adbbfec659f243f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235292615 2024-12-03T14:14:55,389 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e940c827d84dcb8794ce9ebfdf9eea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235292615 2024-12-03T14:14:55,389 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f964e747c0284759a42165099249b165, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:55,389 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52f938a88174796b29b7f30e754d695, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:55,395 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:55,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:55,396 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/c1289286ca7c4d53a8f2f4a072185a81 is 50, key is test_row_0/B:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:55,396 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e98f9dc493fa4905bbc6d70ad986de10 is 50, key is test_row_0/A:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:55,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742133_1309 (size=12595) 2024-12-03T14:14:55,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742134_1310 (size=12595) 2024-12-03T14:14:55,406 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e98f9dc493fa4905bbc6d70ad986de10 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e98f9dc493fa4905bbc6d70ad986de10 2024-12-03T14:14:55,411 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into e98f9dc493fa4905bbc6d70ad986de10(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:55,411 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:55,411 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=13, startTime=1733235295387; duration=0sec 2024-12-03T14:14:55,411 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:55,411 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:14:55,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:14:55,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:14:55,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:14:55,413 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,413 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/5b9087b964fa4386853660cb2d30e799, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=35.9 K 2024-12-03T14:14:55,413 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9087b964fa4386853660cb2d30e799, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733235291962 2024-12-03T14:14:55,413 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8539d0639b4b4fbcfecad070b37784, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733235292615 2024-12-03T14:14:55,413 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb81b63275ad4a70aaeb9b7ad195dacc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:55,418 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#260 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:55,419 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/23170b3de9374bf8b9216571569932da is 50, key is test_row_0/C:col10/1733235294938/Put/seqid=0 2024-12-03T14:14:55,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742135_1311 (size=12595) 2024-12-03T14:14:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T14:14:55,466 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:55,467 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T14:14:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:55,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:55,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:55,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:55,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:55,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/da98465432034946b0ce0fb0348e3f05 is 50, key is test_row_0/A:col10/1733235295597/Put/seqid=0 2024-12-03T14:14:55,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742136_1312 (size=14541) 2024-12-03T14:14:55,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235355623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235355626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:55,805 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/c1289286ca7c4d53a8f2f4a072185a81 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c1289286ca7c4d53a8f2f4a072185a81 2024-12-03T14:14:55,809 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into c1289286ca7c4d53a8f2f4a072185a81(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:55,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:55,810 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=13, startTime=1733235295387; duration=0sec 2024-12-03T14:14:55,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:55,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:14:55,826 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/23170b3de9374bf8b9216571569932da as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/23170b3de9374bf8b9216571569932da 2024-12-03T14:14:55,829 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into 23170b3de9374bf8b9216571569932da(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:55,829 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:55,829 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=13, startTime=1733235295387; duration=0sec 2024-12-03T14:14:55,829 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:55,829 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:14:55,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:55,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:55,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235355946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235355946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:55,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:56,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/da98465432034946b0ce0fb0348e3f05 2024-12-03T14:14:56,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/c9b226a0d68949a1a741ced5a28be1c9 is 50, key is test_row_0/B:col10/1733235295597/Put/seqid=0 2024-12-03T14:14:56,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742137_1313 (size=12151) 2024-12-03T14:14:56,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:56,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235356256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235356256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:56,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/c9b226a0d68949a1a741ced5a28be1c9 2024-12-03T14:14:56,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/dd691c1ed1cf4680a120b0737795ff5e is 50, key is test_row_0/C:col10/1733235295597/Put/seqid=0 2024-12-03T14:14:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742138_1314 (size=12151) 2024-12-03T14:14:56,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:56,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235356765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235356766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:56,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/dd691c1ed1cf4680a120b0737795ff5e 2024-12-03T14:14:56,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/da98465432034946b0ce0fb0348e3f05 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05 2024-12-03T14:14:56,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05, entries=200, sequenceid=245, filesize=14.2 K 2024-12-03T14:14:56,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/c9b226a0d68949a1a741ced5a28be1c9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9 2024-12-03T14:14:56,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9, entries=150, sequenceid=245, filesize=11.9 K 2024-12-03T14:14:56,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/dd691c1ed1cf4680a120b0737795ff5e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e 2024-12-03T14:14:56,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:56,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:56,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e, entries=150, sequenceid=245, filesize=11.9 K 2024-12-03T14:14:56,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d37984c6fe08d63ee0981270943fab76 in 1248ms, sequenceid=245, compaction requested=false 2024-12-03T14:14:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:56,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:56,995 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:56,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:56,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:56,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/69bf44c4a7ce491cb9f8acaf1febeb5e is 50, key is test_row_0/A:col10/1733235295602/Put/seqid=0 2024-12-03T14:14:57,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742139_1315 (size=12251) 2024-12-03T14:14:57,403 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/69bf44c4a7ce491cb9f8acaf1febeb5e 2024-12-03T14:14:57,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ab06956c6ba94fd7b17f8f96841ada39 is 50, key is test_row_0/B:col10/1733235295602/Put/seqid=0 2024-12-03T14:14:57,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742140_1316 (size=12251) 2024-12-03T14:14:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:57,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235357802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235357803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:57,812 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ab06956c6ba94fd7b17f8f96841ada39 2024-12-03T14:14:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0d82fd7410a84ac787d10ee585bd39c5 is 50, key is test_row_0/C:col10/1733235295602/Put/seqid=0 2024-12-03T14:14:57,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742141_1317 (size=12251) 2024-12-03T14:14:57,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235357916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:57,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235357916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37042 deadline: 1733235358016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37050 deadline: 1733235358016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,019 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:58,018 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37070 deadline: 1733235358035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,038 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in d37984c6fe08d63ee0981270943fab76 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235358126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235358126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,226 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0d82fd7410a84ac787d10ee585bd39c5 2024-12-03T14:14:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/69bf44c4a7ce491cb9f8acaf1febeb5e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e 2024-12-03T14:14:58,234 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T14:14:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/ab06956c6ba94fd7b17f8f96841ada39 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39 2024-12-03T14:14:58,238 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T14:14:58,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0d82fd7410a84ac787d10ee585bd39c5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5 2024-12-03T14:14:58,242 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5, entries=150, sequenceid=262, filesize=12.0 K 2024-12-03T14:14:58,243 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d37984c6fe08d63ee0981270943fab76 in 1248ms, sequenceid=262, compaction requested=true 2024-12-03T14:14:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T14:14:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T14:14:58,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T14:14:58,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7760 sec 2024-12-03T14:14:58,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.7800 sec 2024-12-03T14:14:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:58,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:58,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b46fc20ac6e041e5ba6137ca18a7f9c2 is 50, key is test_row_0/A:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742142_1318 (size=14741) 2024-12-03T14:14:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235358460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235358461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235358566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235358566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235358776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235358776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:58,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b46fc20ac6e041e5ba6137ca18a7f9c2 2024-12-03T14:14:58,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/2a44755fe9804081994c5af569384f3c is 50, key is test_row_0/B:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742143_1319 (size=12301) 2024-12-03T14:14:58,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/2a44755fe9804081994c5af569384f3c 2024-12-03T14:14:58,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/af03ba89f26a46c5b2eb3b30c549613a is 50, key is test_row_0/C:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742144_1320 (size=12301) 2024-12-03T14:14:58,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/af03ba89f26a46c5b2eb3b30c549613a 2024-12-03T14:14:58,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/b46fc20ac6e041e5ba6137ca18a7f9c2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2 2024-12-03T14:14:58,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2, entries=200, sequenceid=281, filesize=14.4 K 2024-12-03T14:14:58,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/2a44755fe9804081994c5af569384f3c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c 2024-12-03T14:14:58,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T14:14:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/af03ba89f26a46c5b2eb3b30c549613a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a 2024-12-03T14:14:58,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T14:14:58,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d37984c6fe08d63ee0981270943fab76 in 461ms, sequenceid=281, compaction requested=true 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:58,900 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:14:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:58,900 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54128 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:14:58,901 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:58,901 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:58,901 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e98f9dc493fa4905bbc6d70ad986de10, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=52.9 K 2024-12-03T14:14:58,901 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c1289286ca7c4d53a8f2f4a072185a81, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=48.1 K 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e98f9dc493fa4905bbc6d70ad986de10, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c1289286ca7c4d53a8f2f4a072185a81, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:58,901 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting da98465432034946b0ce0fb0348e3f05, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733235294953 2024-12-03T14:14:58,902 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b226a0d68949a1a741ced5a28be1c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733235294953 2024-12-03T14:14:58,902 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69bf44c4a7ce491cb9f8acaf1febeb5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235295602 2024-12-03T14:14:58,902 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ab06956c6ba94fd7b17f8f96841ada39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235295602 2024-12-03T14:14:58,902 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46fc20ac6e041e5ba6137ca18a7f9c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297796 2024-12-03T14:14:58,902 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a44755fe9804081994c5af569384f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297800 2024-12-03T14:14:58,914 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:58,914 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/bafd10d079bb47c7a9f1150d20806b23 is 50, key is test_row_0/B:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,923 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:58,923 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/c98078f5a77545579b8d85839ddeef9b is 50, key is test_row_0/A:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742145_1321 (size=12881) 2024-12-03T14:14:58,941 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/c98078f5a77545579b8d85839ddeef9b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c98078f5a77545579b8d85839ddeef9b 2024-12-03T14:14:58,946 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into c98078f5a77545579b8d85839ddeef9b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:58,946 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:58,946 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=12, startTime=1733235298900; duration=0sec 2024-12-03T14:14:58,946 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:14:58,946 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:14:58,946 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:14:58,948 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:14:58,948 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:14:58,949 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:58,950 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/23170b3de9374bf8b9216571569932da, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=48.1 K 2024-12-03T14:14:58,950 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23170b3de9374bf8b9216571569932da, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733235293790 2024-12-03T14:14:58,951 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd691c1ed1cf4680a120b0737795ff5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733235294953 2024-12-03T14:14:58,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742146_1322 (size=12881) 2024-12-03T14:14:58,951 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d82fd7410a84ac787d10ee585bd39c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733235295602 2024-12-03T14:14:58,951 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting af03ba89f26a46c5b2eb3b30c549613a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297800 2024-12-03T14:14:58,958 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:14:58,959 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a4ddeb4cecbe4375bc38fd699ffbef28 is 50, key is test_row_0/C:col10/1733235298437/Put/seqid=0 2024-12-03T14:14:58,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742147_1323 (size=12881) 2024-12-03T14:14:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:59,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/d09c41023caa4424be0f44f1b57e7173 is 50, key is test_row_0/A:col10/1733235299087/Put/seqid=0 2024-12-03T14:14:59,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742148_1324 (size=14741) 2024-12-03T14:14:59,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/d09c41023caa4424be0f44f1b57e7173 2024-12-03T14:14:59,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/1beac75a7c48415d9976ceb152354fae is 50, key is test_row_0/B:col10/1733235299087/Put/seqid=0 2024-12-03T14:14:59,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742149_1325 (size=12301) 2024-12-03T14:14:59,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/1beac75a7c48415d9976ceb152354fae 2024-12-03T14:14:59,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/22fa812796c1419191dd62015000d03a is 50, key is test_row_0/C:col10/1733235299087/Put/seqid=0 2024-12-03T14:14:59,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742150_1326 (size=12301) 2024-12-03T14:14:59,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/22fa812796c1419191dd62015000d03a 2024-12-03T14:14:59,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235359130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235359131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/d09c41023caa4424be0f44f1b57e7173 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173 2024-12-03T14:14:59,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173, entries=200, sequenceid=297, filesize=14.4 K 2024-12-03T14:14:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/1beac75a7c48415d9976ceb152354fae as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae 2024-12-03T14:14:59,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T14:14:59,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/22fa812796c1419191dd62015000d03a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a 2024-12-03T14:14:59,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a, entries=150, sequenceid=297, filesize=12.0 K 2024-12-03T14:14:59,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d37984c6fe08d63ee0981270943fab76 in 62ms, sequenceid=297, compaction requested=false 2024-12-03T14:14:59,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:14:59,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:14:59,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:14:59,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/43e6c52a69ed4a68a59283921b1b221a is 50, key is test_row_0/A:col10/1733235299122/Put/seqid=0 2024-12-03T14:14:59,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742151_1327 (size=14741) 2024-12-03T14:14:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235359284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235359284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,356 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/bafd10d079bb47c7a9f1150d20806b23 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/bafd10d079bb47c7a9f1150d20806b23 2024-12-03T14:14:59,361 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into bafd10d079bb47c7a9f1150d20806b23(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:59,361 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:59,361 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=12, startTime=1733235298900; duration=0sec 2024-12-03T14:14:59,361 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:59,361 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:14:59,375 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/a4ddeb4cecbe4375bc38fd699ffbef28 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a4ddeb4cecbe4375bc38fd699ffbef28 2024-12-03T14:14:59,383 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into a4ddeb4cecbe4375bc38fd699ffbef28(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:14:59,383 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:14:59,383 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=12, startTime=1733235298900; duration=0sec 2024-12-03T14:14:59,384 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:14:59,384 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:14:59,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235359396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235359396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235359606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235359606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T14:14:59,636 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:14:59,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:14:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-03T14:14:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:14:59,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:14:59,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:14:59,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:14:59,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/43e6c52a69ed4a68a59283921b1b221a 2024-12-03T14:14:59,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/b31991cb657a42528956763a92d127a0 is 50, key is test_row_0/B:col10/1733235299122/Put/seqid=0 2024-12-03T14:14:59,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742152_1328 (size=12301) 2024-12-03T14:14:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:14:59,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:14:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:59,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235359916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235359916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:14:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:14:59,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:14:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:14:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:14:59,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:14:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:15:00,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/b31991cb657a42528956763a92d127a0 2024-12-03T14:15:00,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/394edd6babd546c3aaa2eec071183789 is 50, key is test_row_0/C:col10/1733235299122/Put/seqid=0 2024-12-03T14:15:00,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742153_1329 (size=12301) 2024-12-03T14:15:00,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:15:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:15:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:15:00,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:15:00,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:00,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:00,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235360426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:00,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235360426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:00,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:00,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/394edd6babd546c3aaa2eec071183789 2024-12-03T14:15:00,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/43e6c52a69ed4a68a59283921b1b221a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a 2024-12-03T14:15:00,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a, entries=200, sequenceid=316, filesize=14.4 K 2024-12-03T14:15:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/b31991cb657a42528956763a92d127a0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0 2024-12-03T14:15:00,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T14:15:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/394edd6babd546c3aaa2eec071183789 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789 2024-12-03T14:15:00,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789, entries=150, sequenceid=316, filesize=12.0 K 2024-12-03T14:15:00,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d37984c6fe08d63ee0981270943fab76 in 1254ms, sequenceid=316, compaction requested=true 2024-12-03T14:15:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:00,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:00,503 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:00,503 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:00,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:00,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:00,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:00,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:00,503 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:00,503 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:15:00,504 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,504 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,504 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c98078f5a77545579b8d85839ddeef9b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=41.4 K 2024-12-03T14:15:00,504 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/bafd10d079bb47c7a9f1150d20806b23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=36.6 K 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting bafd10d079bb47c7a9f1150d20806b23, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297800 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98078f5a77545579b8d85839ddeef9b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297800 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09c41023caa4424be0f44f1b57e7173, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733235298442 2024-12-03T14:15:00,504 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1beac75a7c48415d9976ceb152354fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733235298459 2024-12-03T14:15:00,505 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e6c52a69ed4a68a59283921b1b221a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:00,505 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b31991cb657a42528956763a92d127a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:00,511 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:00,512 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/bfb0490532314a9bbdd068f9ce679c39 is 50, key is test_row_0/A:col10/1733235299122/Put/seqid=0 2024-12-03T14:15:00,514 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:00,515 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/d4efd1d9ce3b44ceb6387ea31d180f31 is 50, key is test_row_0/B:col10/1733235299122/Put/seqid=0 2024-12-03T14:15:00,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742154_1330 (size=12983) 2024-12-03T14:15:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742155_1331 (size=12983) 2024-12-03T14:15:00,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T14:15:00,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,555 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:15:00,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:00,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/4cb9c51fca4b4a6f9cbb11d65016ef23 is 50, key is test_row_0/A:col10/1733235299279/Put/seqid=0 2024-12-03T14:15:00,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742156_1332 (size=12301) 2024-12-03T14:15:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:15:00,934 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/bfb0490532314a9bbdd068f9ce679c39 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bfb0490532314a9bbdd068f9ce679c39 2024-12-03T14:15:00,937 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into bfb0490532314a9bbdd068f9ce679c39(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:00,938 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:00,938 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=13, startTime=1733235300502; duration=0sec 2024-12-03T14:15:00,938 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:00,938 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:15:00,938 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:00,939 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:00,939 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:15:00,939 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:00,939 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a4ddeb4cecbe4375bc38fd699ffbef28, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=36.6 K 2024-12-03T14:15:00,940 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ddeb4cecbe4375bc38fd699ffbef28, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733235297800 2024-12-03T14:15:00,940 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22fa812796c1419191dd62015000d03a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733235298459 2024-12-03T14:15:00,940 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394edd6babd546c3aaa2eec071183789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:00,943 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/d4efd1d9ce3b44ceb6387ea31d180f31 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/d4efd1d9ce3b44ceb6387ea31d180f31 2024-12-03T14:15:00,945 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#282 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:00,946 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/6500045da7ec4b24a8a5c9a478c3a2df is 50, key is test_row_0/C:col10/1733235299122/Put/seqid=0 2024-12-03T14:15:00,947 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into d4efd1d9ce3b44ceb6387ea31d180f31(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:00,947 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:00,947 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=13, startTime=1733235300503; duration=0sec 2024-12-03T14:15:00,947 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:00,948 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:15:00,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742157_1333 (size=12983) 2024-12-03T14:15:00,966 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/4cb9c51fca4b4a6f9cbb11d65016ef23 2024-12-03T14:15:00,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/37446d6fbbf240b3961e7ec84af3b2fd is 50, key is test_row_0/B:col10/1733235299279/Put/seqid=0 2024-12-03T14:15:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742158_1334 (size=12301) 2024-12-03T14:15:00,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/37446d6fbbf240b3961e7ec84af3b2fd 2024-12-03T14:15:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/50d5c931ba1b41b19ba6f63640beeaab is 50, key is test_row_0/C:col10/1733235299279/Put/seqid=0 2024-12-03T14:15:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742159_1335 (size=12301) 2024-12-03T14:15:01,353 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/6500045da7ec4b24a8a5c9a478c3a2df as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/6500045da7ec4b24a8a5c9a478c3a2df 2024-12-03T14:15:01,357 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into 6500045da7ec4b24a8a5c9a478c3a2df(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:01,357 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:01,357 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=13, startTime=1733235300503; duration=0sec 2024-12-03T14:15:01,357 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:01,357 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:15:01,397 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/50d5c931ba1b41b19ba6f63640beeaab 2024-12-03T14:15:01,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/4cb9c51fca4b4a6f9cbb11d65016ef23 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23 2024-12-03T14:15:01,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23, entries=150, sequenceid=331, filesize=12.0 K 2024-12-03T14:15:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/37446d6fbbf240b3961e7ec84af3b2fd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd 2024-12-03T14:15:01,418 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd, entries=150, sequenceid=331, filesize=12.0 K 2024-12-03T14:15:01,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/50d5c931ba1b41b19ba6f63640beeaab as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab 2024-12-03T14:15:01,428 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab, entries=150, sequenceid=331, filesize=12.0 K 2024-12-03T14:15:01,429 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for d37984c6fe08d63ee0981270943fab76 in 874ms, sequenceid=331, compaction requested=false 2024-12-03T14:15:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-03T14:15:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-03T14:15:01,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T14:15:01,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-12-03T14:15:01,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7970 sec 2024-12-03T14:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:01,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:15:01,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:01,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/3c4736fb0a914b5b9e771ce7f58328bc is 50, key is test_row_0/A:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:01,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742160_1336 (size=12301) 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235301505, max timestamp=1733235301514Num uniques 859; 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235301505, max timestamp=1733235301514Num uniques 859; 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235301505, max timestamp=1733235301514Num uniques 859; 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:01,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235361553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235361557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235361666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235361666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T14:15:01,787 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:01,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-03T14:15:01,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:01,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:01,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235361876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235361876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:01,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/3c4736fb0a914b5b9e771ce7f58328bc 2024-12-03T14:15:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:01,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/413d27ab952d4a91b2c187093d151660 is 50, key is test_row_0/B:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:01,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742161_1337 (size=12301) 2024-12-03T14:15:01,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:01,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:01,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:02,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235362186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:02,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235362186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:02,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/413d27ab952d4a91b2c187093d151660 2024-12-03T14:15:02,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/416b70566ff74cd7b8bfcb3b1ec5aa13 is 50, key is test_row_0/C:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:02,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742162_1338 (size=12301) 2024-12-03T14:15:02,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:02,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37036 deadline: 1733235362696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37028 deadline: 1733235362696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:02,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:02,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:02,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/416b70566ff74cd7b8bfcb3b1ec5aa13 2024-12-03T14:15:02,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/3c4736fb0a914b5b9e771ce7f58328bc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc 2024-12-03T14:15:02,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T14:15:02,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/413d27ab952d4a91b2c187093d151660 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660 2024-12-03T14:15:02,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T14:15:02,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/416b70566ff74cd7b8bfcb3b1ec5aa13 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13 2024-12-03T14:15:02,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T14:15:02,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d37984c6fe08d63ee0981270943fab76 in 1267ms, sequenceid=345, compaction requested=true 2024-12-03T14:15:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:02,735 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:02,736 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d37984c6fe08d63ee0981270943fab76:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:02,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:02,736 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:02,736 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/A is initiating minor compaction (all files) 2024-12-03T14:15:02,737 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/A in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,737 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bfb0490532314a9bbdd068f9ce679c39, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=36.7 K 2024-12-03T14:15:02,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:02,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/B is initiating minor compaction (all files) 2024-12-03T14:15:02,737 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/B in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,737 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/d4efd1d9ce3b44ceb6387ea31d180f31, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=36.7 K 2024-12-03T14:15:02,737 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfb0490532314a9bbdd068f9ce679c39, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:02,737 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d4efd1d9ce3b44ceb6387ea31d180f31, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:02,737 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb9c51fca4b4a6f9cbb11d65016ef23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733235299251 2024-12-03T14:15:02,738 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 37446d6fbbf240b3961e7ec84af3b2fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733235299251 2024-12-03T14:15:02,738 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4736fb0a914b5b9e771ce7f58328bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733235301438 2024-12-03T14:15:02,738 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 413d27ab952d4a91b2c187093d151660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733235301438 2024-12-03T14:15:02,744 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:02,744 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:02,744 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/635b931369d044eb98885560a43124eb is 50, key is test_row_0/A:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:02,744 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/e9d31f140e134f7182e2980e73ab273f is 50, key is test_row_0/B:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:02,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742163_1339 (size=13085) 2024-12-03T14:15:02,763 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/e9d31f140e134f7182e2980e73ab273f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e9d31f140e134f7182e2980e73ab273f 2024-12-03T14:15:02,768 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/B of d37984c6fe08d63ee0981270943fab76 into e9d31f140e134f7182e2980e73ab273f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:02,768 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:02,768 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/B, priority=13, startTime=1733235302736; duration=0sec 2024-12-03T14:15:02,769 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:02,769 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:B 2024-12-03T14:15:02,769 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:02,770 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:02,770 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): d37984c6fe08d63ee0981270943fab76/C is initiating minor compaction (all files) 2024-12-03T14:15:02,770 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d37984c6fe08d63ee0981270943fab76/C in TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,770 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/6500045da7ec4b24a8a5c9a478c3a2df, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp, totalSize=36.7 K 2024-12-03T14:15:02,770 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6500045da7ec4b24a8a5c9a478c3a2df, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733235299115 2024-12-03T14:15:02,770 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d5c931ba1b41b19ba6f63640beeaab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733235299251 2024-12-03T14:15:02,771 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 416b70566ff74cd7b8bfcb3b1ec5aa13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733235301438 2024-12-03T14:15:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742164_1340 (size=13085) 2024-12-03T14:15:02,787 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d37984c6fe08d63ee0981270943fab76#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:02,788 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0c1c641f773541f3bed6d92d7a1002b1 is 50, key is test_row_0/C:col10/1733235301467/Put/seqid=0 2024-12-03T14:15:02,789 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/635b931369d044eb98885560a43124eb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/635b931369d044eb98885560a43124eb 2024-12-03T14:15:02,795 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/A of d37984c6fe08d63ee0981270943fab76 into 635b931369d044eb98885560a43124eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:02,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:02,795 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/A, priority=13, startTime=1733235302735; duration=0sec 2024-12-03T14:15:02,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:02,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:A 2024-12-03T14:15:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742165_1341 (size=13085) 2024-12-03T14:15:02,809 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/0c1c641f773541f3bed6d92d7a1002b1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c1c641f773541f3bed6d92d7a1002b1 2024-12-03T14:15:02,816 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d37984c6fe08d63ee0981270943fab76/C of d37984c6fe08d63ee0981270943fab76 into 0c1c641f773541f3bed6d92d7a1002b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:02,816 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:02,816 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., storeName=d37984c6fe08d63ee0981270943fab76/C, priority=13, startTime=1733235302736; duration=0sec 2024-12-03T14:15:02,816 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:02,816 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d37984c6fe08d63ee0981270943fab76:C 2024-12-03T14:15:02,823 INFO [Thread-1252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1252. 2024-12-03T14:15:02,823 INFO [Thread-1250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1250. 2024-12-03T14:15:02,824 DEBUG [Thread-1250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:02,824 DEBUG [Thread-1252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:02,824 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,824 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,824 DEBUG [Thread-1250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,824 DEBUG [Thread-1252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:02,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:02,825 INFO [Thread-1246 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1246. 2024-12-03T14:15:02,825 DEBUG [Thread-1246 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:02,825 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,825 DEBUG [Thread-1246 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:02,826 INFO [Thread-1244 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1244. 2024-12-03T14:15:02,826 DEBUG [Thread-1244 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:02,826 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,826 DEBUG [Thread-1244 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:02,826 INFO [Thread-1248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1248. 2024-12-03T14:15:02,826 DEBUG [Thread-1248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:02,826 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,826 DEBUG [Thread-1248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:02,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:02,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:02,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:15:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/9311a9418ae2482cb298a77111b2c2c8 is 50, key is test_row_0/A:col10/1733235301555/Put/seqid=0 2024-12-03T14:15:02,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742166_1342 (size=12301) 2024-12-03T14:15:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:03,271 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/9311a9418ae2482cb298a77111b2c2c8 2024-12-03T14:15:03,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/765891b547994f6fb9ee52ecec39a50b is 50, key is test_row_0/B:col10/1733235301555/Put/seqid=0 2024-12-03T14:15:03,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742167_1343 (size=12301) 2024-12-03T14:15:03,683 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/765891b547994f6fb9ee52ecec39a50b 2024-12-03T14:15:03,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/8cdf6795443246969e3a031cd90c5703 is 50, key is test_row_0/C:col10/1733235301555/Put/seqid=0 2024-12-03T14:15:03,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742168_1344 (size=12301) 2024-12-03T14:15:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:03,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. as already flushing 2024-12-03T14:15:03,708 INFO [Thread-1239 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1239. 2024-12-03T14:15:03,708 INFO [Thread-1237 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1237. 2024-12-03T14:15:03,708 DEBUG [Thread-1237 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:03,708 DEBUG [Thread-1239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:03,708 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:03,708 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:03,708 DEBUG [Thread-1237 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:03,708 DEBUG [Thread-1239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:03,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:03,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:04,095 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/8cdf6795443246969e3a031cd90c5703 2024-12-03T14:15:04,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/9311a9418ae2482cb298a77111b2c2c8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/9311a9418ae2482cb298a77111b2c2c8 2024-12-03T14:15:04,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/9311a9418ae2482cb298a77111b2c2c8, entries=150, sequenceid=384, filesize=12.0 K 2024-12-03T14:15:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/765891b547994f6fb9ee52ecec39a50b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/765891b547994f6fb9ee52ecec39a50b 2024-12-03T14:15:04,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/765891b547994f6fb9ee52ecec39a50b, entries=150, sequenceid=384, filesize=12.0 K 2024-12-03T14:15:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/8cdf6795443246969e3a031cd90c5703 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/8cdf6795443246969e3a031cd90c5703 2024-12-03T14:15:04,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/8cdf6795443246969e3a031cd90c5703, entries=150, sequenceid=384, filesize=12.0 K 2024-12-03T14:15:04,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=13.42 KB/13740 for d37984c6fe08d63ee0981270943fab76 in 1249ms, sequenceid=384, compaction requested=false 2024-12-03T14:15:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d37984c6fe08d63ee0981270943fab76: 2024-12-03T14:15:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-03T14:15:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-03T14:15:04,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T14:15:04,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-12-03T14:15:04,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3270 sec 2024-12-03T14:15:05,458 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:15:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T14:15:05,946 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:08,048 INFO [Thread-1235 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1235. 2024-12-03T14:15:08,048 INFO [Thread-1241 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1241. 2024-12-03T14:15:08,048 DEBUG [Thread-1235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:08,048 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,048 DEBUG [Thread-1241 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:08,048 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,049 DEBUG [Thread-1235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,049 DEBUG [Thread-1241 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:08,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:08,118 INFO [Thread-1233 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1233. 2024-12-03T14:15:08,118 DEBUG [Thread-1233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:08,118 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,118 DEBUG [Thread-1233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4679 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4542 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4664 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4706 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4531 2024-12-03T14:15:08,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:15:08,119 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:15:08,119 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:15:08,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:08,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:08,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:15:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:08,122 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235308122"}]},"ts":"1733235308122"} 2024-12-03T14:15:08,123 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:15:08,123 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:15:08,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:15:08,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, UNASSIGN}] 2024-12-03T14:15:08,126 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, UNASSIGN 2024-12-03T14:15:08,126 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=d37984c6fe08d63ee0981270943fab76, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:08,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, UNASSIGN because future has completed 2024-12-03T14:15:08,127 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:15:08,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:08,280 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing d37984c6fe08d63ee0981270943fab76, disabling compactions & flushes 2024-12-03T14:15:08,280 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. after waiting 0 ms 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:08,280 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing d37984c6fe08d63ee0981270943fab76 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=A 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=B 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d37984c6fe08d63ee0981270943fab76, store=C 2024-12-03T14:15:08,280 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:08,284 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e36ee5da86094d9e90a9bf70e9e612c0 is 50, key is test_row_0/A:col10/1733235308047/Put/seqid=0 2024-12-03T14:15:08,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742169_1345 (size=9857) 2024-12-03T14:15:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:08,689 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e36ee5da86094d9e90a9bf70e9e612c0 2024-12-03T14:15:08,701 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/4279fd5d8dd647a3824a4c35d8ec72c4 is 50, key is test_row_0/B:col10/1733235308047/Put/seqid=0 2024-12-03T14:15:08,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742170_1346 (size=9857) 2024-12-03T14:15:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:09,106 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/4279fd5d8dd647a3824a4c35d8ec72c4 2024-12-03T14:15:09,118 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/47401249db3f4400b6e70dac31dc8840 is 50, key is test_row_0/C:col10/1733235308047/Put/seqid=0 2024-12-03T14:15:09,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742171_1347 (size=9857) 2024-12-03T14:15:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:09,524 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/47401249db3f4400b6e70dac31dc8840 2024-12-03T14:15:09,529 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/A/e36ee5da86094d9e90a9bf70e9e612c0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e36ee5da86094d9e90a9bf70e9e612c0 2024-12-03T14:15:09,533 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e36ee5da86094d9e90a9bf70e9e612c0, entries=100, sequenceid=392, filesize=9.6 K 2024-12-03T14:15:09,534 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/B/4279fd5d8dd647a3824a4c35d8ec72c4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/4279fd5d8dd647a3824a4c35d8ec72c4 2024-12-03T14:15:09,537 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/4279fd5d8dd647a3824a4c35d8ec72c4, entries=100, sequenceid=392, filesize=9.6 K 2024-12-03T14:15:09,537 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/.tmp/C/47401249db3f4400b6e70dac31dc8840 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/47401249db3f4400b6e70dac31dc8840 2024-12-03T14:15:09,540 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/47401249db3f4400b6e70dac31dc8840, entries=100, sequenceid=392, filesize=9.6 K 2024-12-03T14:15:09,540 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d37984c6fe08d63ee0981270943fab76 in 1260ms, sequenceid=392, compaction requested=true 2024-12-03T14:15:09,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/320c28d9b18145d281cfb348de681dfc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/5f240c908fdd4f2c8a7cad02b1279173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/ae0469b47d90422d9f4a8515f4734e8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e98f9dc493fa4905bbc6d70ad986de10, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c98078f5a77545579b8d85839ddeef9b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bfb0490532314a9bbdd068f9ce679c39, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc] to archive 2024-12-03T14:15:09,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:09,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bff88f4f3bfc42a19f680a39279e9c40 2024-12-03T14:15:09,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/38c1a358accf4f178949a765c53a1c76 2024-12-03T14:15:09,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/95af8d647d5a4c538c0f4544b213c651 2024-12-03T14:15:09,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/320c28d9b18145d281cfb348de681dfc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/320c28d9b18145d281cfb348de681dfc 2024-12-03T14:15:09,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/51052213e589440dba033f4e2dc9bbbd 2024-12-03T14:15:09,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/41184cffc2f14823ba013d8022f0e305 2024-12-03T14:15:09,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/5f240c908fdd4f2c8a7cad02b1279173 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/5f240c908fdd4f2c8a7cad02b1279173 2024-12-03T14:15:09,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c8587ac0aaaa465e8f8d747aa726a40b 2024-12-03T14:15:09,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/f9d271691b3e406e90c09b3273736603 2024-12-03T14:15:09,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b60f046045074fa3800c331687795f26 2024-12-03T14:15:09,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/ae0469b47d90422d9f4a8515f4734e8e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/ae0469b47d90422d9f4a8515f4734e8e 2024-12-03T14:15:09,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/a3e940c827d84dcb8794ce9ebfdf9eea 2024-12-03T14:15:09,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e52f938a88174796b29b7f30e754d695 2024-12-03T14:15:09,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e98f9dc493fa4905bbc6d70ad986de10 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e98f9dc493fa4905bbc6d70ad986de10 2024-12-03T14:15:09,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/da98465432034946b0ce0fb0348e3f05 2024-12-03T14:15:09,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/69bf44c4a7ce491cb9f8acaf1febeb5e 2024-12-03T14:15:09,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/b46fc20ac6e041e5ba6137ca18a7f9c2 2024-12-03T14:15:09,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c98078f5a77545579b8d85839ddeef9b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/c98078f5a77545579b8d85839ddeef9b 2024-12-03T14:15:09,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/d09c41023caa4424be0f44f1b57e7173 2024-12-03T14:15:09,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/43e6c52a69ed4a68a59283921b1b221a 2024-12-03T14:15:09,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bfb0490532314a9bbdd068f9ce679c39 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/bfb0490532314a9bbdd068f9ce679c39 2024-12-03T14:15:09,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/4cb9c51fca4b4a6f9cbb11d65016ef23 2024-12-03T14:15:09,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/3c4736fb0a914b5b9e771ce7f58328bc 2024-12-03T14:15:09,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2fd351e72a174df68965c39ec7099d43, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/387efaa120884dd599d892c5abe4ff89, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/9a175a911c3a4db79a197cfc55f757af, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c1289286ca7c4d53a8f2f4a072185a81, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/bafd10d079bb47c7a9f1150d20806b23, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/d4efd1d9ce3b44ceb6387ea31d180f31, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660] to archive 2024-12-03T14:15:09,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:09,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/7a97e0514ecf44179fc1e98f3d3228ed 2024-12-03T14:15:09,563 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/fc195c78c9f34629bf557144ccbe58e6 2024-12-03T14:15:09,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2fd351e72a174df68965c39ec7099d43 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2fd351e72a174df68965c39ec7099d43 2024-12-03T14:15:09,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f94bac6b08174e3d8478fb7700c55ec5 2024-12-03T14:15:09,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e0572505dda04865ba7a103685a1cf8e 2024-12-03T14:15:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ed6c3cdbfd1848cd9b0643ca3ec24f04 2024-12-03T14:15:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/387efaa120884dd599d892c5abe4ff89 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/387efaa120884dd599d892c5abe4ff89 2024-12-03T14:15:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/61045bae5608472bbe6610036291be52 2024-12-03T14:15:09,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/3a4dadddbcfe45719eca7cc6d563759c 2024-12-03T14:15:09,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/9a175a911c3a4db79a197cfc55f757af to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/9a175a911c3a4db79a197cfc55f757af 2024-12-03T14:15:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/0fec4cf9bca14e95b475c760b348502d 2024-12-03T14:15:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/5bd81e1b229848f68adbbfec659f243f 2024-12-03T14:15:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c1289286ca7c4d53a8f2f4a072185a81 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c1289286ca7c4d53a8f2f4a072185a81 2024-12-03T14:15:09,571 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/f964e747c0284759a42165099249b165 2024-12-03T14:15:09,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/c9b226a0d68949a1a741ced5a28be1c9 2024-12-03T14:15:09,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/ab06956c6ba94fd7b17f8f96841ada39 2024-12-03T14:15:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/bafd10d079bb47c7a9f1150d20806b23 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/bafd10d079bb47c7a9f1150d20806b23 2024-12-03T14:15:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/2a44755fe9804081994c5af569384f3c 2024-12-03T14:15:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/1beac75a7c48415d9976ceb152354fae 2024-12-03T14:15:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/d4efd1d9ce3b44ceb6387ea31d180f31 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/d4efd1d9ce3b44ceb6387ea31d180f31 2024-12-03T14:15:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/b31991cb657a42528956763a92d127a0 2024-12-03T14:15:09,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/37446d6fbbf240b3961e7ec84af3b2fd 2024-12-03T14:15:09,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/413d27ab952d4a91b2c187093d151660 2024-12-03T14:15:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7b35d9dd4a504ff08d6c40bf790fa561, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/d642bcc424014305913d2afe96324674, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/5b9087b964fa4386853660cb2d30e799, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/23170b3de9374bf8b9216571569932da, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a4ddeb4cecbe4375bc38fd699ffbef28, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/6500045da7ec4b24a8a5c9a478c3a2df, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13] to archive 2024-12-03T14:15:09,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:09,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a141cb84e7da4f778c2413d9e2a0dcbc 2024-12-03T14:15:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/f8b310d0270f49da956e289c7d8dfe25 2024-12-03T14:15:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7b35d9dd4a504ff08d6c40bf790fa561 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7b35d9dd4a504ff08d6c40bf790fa561 2024-12-03T14:15:09,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a454f82bbb9545cc9adc957807730c6a 2024-12-03T14:15:09,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/ac9c37a3db4a48d9bdc95d495d23c92d 2024-12-03T14:15:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/c6fcf035c0a04717a87a302421b58b8e 2024-12-03T14:15:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/d642bcc424014305913d2afe96324674 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/d642bcc424014305913d2afe96324674 2024-12-03T14:15:09,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/64148c97b5764bad849e57aa69e04c6c 2024-12-03T14:15:09,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/7e65157aebdb4012aa5ad4fc4519b17d 2024-12-03T14:15:09,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/5b9087b964fa4386853660cb2d30e799 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/5b9087b964fa4386853660cb2d30e799 2024-12-03T14:15:09,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a2712ad021244b88b91dd4bab855d521 2024-12-03T14:15:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c8539d0639b4b4fbcfecad070b37784 2024-12-03T14:15:09,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/23170b3de9374bf8b9216571569932da to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/23170b3de9374bf8b9216571569932da 2024-12-03T14:15:09,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/bb81b63275ad4a70aaeb9b7ad195dacc 2024-12-03T14:15:09,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/dd691c1ed1cf4680a120b0737795ff5e 2024-12-03T14:15:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0d82fd7410a84ac787d10ee585bd39c5 2024-12-03T14:15:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a4ddeb4cecbe4375bc38fd699ffbef28 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/a4ddeb4cecbe4375bc38fd699ffbef28 2024-12-03T14:15:09,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/af03ba89f26a46c5b2eb3b30c549613a 2024-12-03T14:15:09,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/22fa812796c1419191dd62015000d03a 2024-12-03T14:15:09,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/6500045da7ec4b24a8a5c9a478c3a2df to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/6500045da7ec4b24a8a5c9a478c3a2df 2024-12-03T14:15:09,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/394edd6babd546c3aaa2eec071183789 2024-12-03T14:15:09,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/50d5c931ba1b41b19ba6f63640beeaab 2024-12-03T14:15:09,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/416b70566ff74cd7b8bfcb3b1ec5aa13 2024-12-03T14:15:09,599 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=1 2024-12-03T14:15:09,599 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. 2024-12-03T14:15:09,599 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for d37984c6fe08d63ee0981270943fab76: Waiting for close lock at 1733235308280Running coprocessor pre-close hooks at 1733235308280Disabling compacts and flushes for region at 1733235308280Disabling writes for close at 1733235308280Obtaining lock to block concurrent updates at 1733235308280Preparing flush snapshotting stores in d37984c6fe08d63ee0981270943fab76 at 1733235308280Finished memstore snapshotting TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733235308280Flushing stores of TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76. at 1733235308281 (+1 ms)Flushing d37984c6fe08d63ee0981270943fab76/A: creating writer at 1733235308281Flushing d37984c6fe08d63ee0981270943fab76/A: appending metadata at 1733235308284 (+3 ms)Flushing d37984c6fe08d63ee0981270943fab76/A: closing flushed file at 1733235308284Flushing d37984c6fe08d63ee0981270943fab76/B: creating writer at 1733235308695 (+411 ms)Flushing d37984c6fe08d63ee0981270943fab76/B: appending metadata at 1733235308700 (+5 ms)Flushing d37984c6fe08d63ee0981270943fab76/B: closing flushed file at 1733235308700Flushing d37984c6fe08d63ee0981270943fab76/C: creating writer at 1733235309113 (+413 ms)Flushing d37984c6fe08d63ee0981270943fab76/C: appending metadata at 1733235309117 (+4 ms)Flushing d37984c6fe08d63ee0981270943fab76/C: closing flushed file at 1733235309117Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58fdb085: reopening flushed file at 1733235309528 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@759985d6: reopening flushed file at 1733235309533 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@440f1fc2: reopening flushed file at 1733235309537 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d37984c6fe08d63ee0981270943fab76 in 1260ms, sequenceid=392, compaction requested=true at 1733235309540 (+3 ms)Writing region close event to WAL at 1733235309596 (+56 ms)Running coprocessor post-close hooks at 1733235309599 (+3 ms)Closed at 1733235309599 2024-12-03T14:15:09,601 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:09,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=d37984c6fe08d63ee0981270943fab76, regionState=CLOSED 2024-12-03T14:15:09,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:09,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T14:15:09,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure d37984c6fe08d63ee0981270943fab76, server=3920e6a7c3b5,42177,1733235217010 in 1.4760 sec 2024-12-03T14:15:09,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T14:15:09,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d37984c6fe08d63ee0981270943fab76, UNASSIGN in 1.4790 sec 2024-12-03T14:15:09,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T14:15:09,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4820 sec 2024-12-03T14:15:09,608 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235309608"}]},"ts":"1733235309608"} 2024-12-03T14:15:09,609 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:15:09,609 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:15:09,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4900 sec 2024-12-03T14:15:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T14:15:10,266 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:10,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,270 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T14:15:10,271 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:10,278 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/recovered.edits] 2024-12-03T14:15:10,282 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/635b931369d044eb98885560a43124eb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/635b931369d044eb98885560a43124eb 2024-12-03T14:15:10,284 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/9311a9418ae2482cb298a77111b2c2c8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/9311a9418ae2482cb298a77111b2c2c8 2024-12-03T14:15:10,285 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e36ee5da86094d9e90a9bf70e9e612c0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/A/e36ee5da86094d9e90a9bf70e9e612c0 2024-12-03T14:15:10,287 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/4279fd5d8dd647a3824a4c35d8ec72c4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/4279fd5d8dd647a3824a4c35d8ec72c4 2024-12-03T14:15:10,288 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/765891b547994f6fb9ee52ecec39a50b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/765891b547994f6fb9ee52ecec39a50b 2024-12-03T14:15:10,289 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e9d31f140e134f7182e2980e73ab273f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/B/e9d31f140e134f7182e2980e73ab273f 2024-12-03T14:15:10,291 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c1c641f773541f3bed6d92d7a1002b1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/0c1c641f773541f3bed6d92d7a1002b1 2024-12-03T14:15:10,292 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/47401249db3f4400b6e70dac31dc8840 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/47401249db3f4400b6e70dac31dc8840 2024-12-03T14:15:10,293 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/8cdf6795443246969e3a031cd90c5703 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/C/8cdf6795443246969e3a031cd90c5703 2024-12-03T14:15:10,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/recovered.edits/395.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76/recovered.edits/395.seqid 2024-12-03T14:15:10,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/d37984c6fe08d63ee0981270943fab76 2024-12-03T14:15:10,296 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:15:10,298 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,301 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:15:10,303 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:15:10,304 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,304 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:15:10,304 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235310304"}]},"ts":"9223372036854775807"} 2024-12-03T14:15:10,306 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:15:10,306 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d37984c6fe08d63ee0981270943fab76, NAME => 'TestAcidGuarantees,,1733235280615.d37984c6fe08d63ee0981270943fab76.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:15:10,306 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:15:10,306 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235310306"}]},"ts":"9223372036854775807"} 2024-12-03T14:15:10,308 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:15:10,308 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-03T14:15:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T14:15:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:15:10,376 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:10,388 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 335) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11020 (was 10087) - AvailableMemoryMB LEAK? - 2024-12-03T14:15:10,396 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=11020 2024-12-03T14:15:10,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:15:10,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:15:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:10,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:15:10,400 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:10,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-03T14:15:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:10,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:15:10,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742172_1348 (size=1061) 2024-12-03T14:15:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:10,812 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:15:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742173_1349 (size=53) 2024-12-03T14:15:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:11,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:15:11,221 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 22c66e6703cd1ab865c50ee7fd3f8725, disabling compactions & flushes 2024-12-03T14:15:11,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. after waiting 0 ms 2024-12-03T14:15:11,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 22c66e6703cd1ab865c50ee7fd3f8725: Waiting for close lock at 1733235311221Disabling compacts and flushes for region at 1733235311221Disabling writes for close at 1733235311222 (+1 ms)Writing region close event to WAL at 1733235311222Closed at 1733235311222 2024-12-03T14:15:11,223 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:15:11,223 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235311223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235311223"}]},"ts":"1733235311223"} 2024-12-03T14:15:11,224 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:15:11,225 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:15:11,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235311225"}]},"ts":"1733235311225"} 2024-12-03T14:15:11,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:15:11,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, ASSIGN}] 2024-12-03T14:15:11,227 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, ASSIGN 2024-12-03T14:15:11,228 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:15:11,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:11,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, ASSIGN because future has completed 2024-12-03T14:15:11,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:11,540 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,540 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:15:11,540 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,541 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:15:11,541 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,541 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,543 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,544 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:11,544 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName A 2024-12-03T14:15:11,544 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:11,545 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:11,545 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,547 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:11,547 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName B 2024-12-03T14:15:11,547 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:11,548 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:11,548 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,549 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:11,549 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName C 2024-12-03T14:15:11,550 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:11,550 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:11,550 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,550 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,551 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,552 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,553 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,553 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,553 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:15:11,555 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,557 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:15:11,558 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 22c66e6703cd1ab865c50ee7fd3f8725; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64162488, jitterRate=-0.04390442371368408}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:15:11,558 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:11,558 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 22c66e6703cd1ab865c50ee7fd3f8725: Running coprocessor pre-open hook at 1733235311541Writing region info on filesystem at 1733235311541Initializing all the Stores at 1733235311542 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235311542Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235311542Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235311542Cleaning up temporary data from old regions at 1733235311553 (+11 ms)Running coprocessor post-open hooks at 1733235311558 (+5 ms)Region opened successfully at 1733235311558 2024-12-03T14:15:11,559 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., pid=87, masterSystemTime=1733235311535 2024-12-03T14:15:11,561 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,561 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:11,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:11,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:11,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T14:15:11,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 in 181 msec 2024-12-03T14:15:11,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T14:15:11,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, ASSIGN in 338 msec 2024-12-03T14:15:11,566 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:15:11,567 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235311566"}]},"ts":"1733235311566"} 2024-12-03T14:15:11,568 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:15:11,568 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:15:11,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-12-03T14:15:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T14:15:12,546 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:12,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:12,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:12,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:12,548 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b275982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:12,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7099575c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:12,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:12,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:12,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:12,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6da78232 2024-12-03T14:15:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:15:12,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:15:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:15:12,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:15:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:12,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742174_1350 (size=1097) 2024-12-03T14:15:12,969 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T14:15:12,969 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T14:15:12,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:15:12,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE}] 2024-12-03T14:15:12,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE 2024-12-03T14:15:12,979 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:12,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE because future has completed 2024-12-03T14:15:12,980 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:15:12,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:13,133 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,133 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:15:13,133 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 22c66e6703cd1ab865c50ee7fd3f8725, disabling compactions & flushes 2024-12-03T14:15:13,133 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,133 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,133 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. after waiting 0 ms 2024-12-03T14:15:13,133 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,137 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T14:15:13,138 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,138 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 22c66e6703cd1ab865c50ee7fd3f8725: Waiting for close lock at 1733235313133Running coprocessor pre-close hooks at 1733235313133Disabling compacts and flushes for region at 1733235313133Disabling writes for close at 1733235313133Writing region close event to WAL at 1733235313134 (+1 ms)Running coprocessor post-close hooks at 1733235313138 (+4 ms)Closed at 1733235313138 2024-12-03T14:15:13,138 WARN [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 22c66e6703cd1ab865c50ee7fd3f8725 to self. 2024-12-03T14:15:13,140 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,140 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=CLOSED 2024-12-03T14:15:13,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:13,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-03T14:15:13,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 in 162 msec 2024-12-03T14:15:13,145 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE; state=CLOSED, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=true 2024-12-03T14:15:13,295 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE because future has completed 2024-12-03T14:15:13,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:13,451 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,451 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:15:13,451 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,451 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:15:13,451 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,451 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,452 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,453 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:13,453 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName A 2024-12-03T14:15:13,454 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:13,455 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:13,455 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,456 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:13,456 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName B 2024-12-03T14:15:13,456 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:13,457 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:13,457 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,457 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:13,457 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22c66e6703cd1ab865c50ee7fd3f8725 columnFamilyName C 2024-12-03T14:15:13,457 DEBUG [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:13,458 INFO [StoreOpener-22c66e6703cd1ab865c50ee7fd3f8725-1 {}] regionserver.HStore(327): Store=22c66e6703cd1ab865c50ee7fd3f8725/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:13,458 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,458 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,459 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,459 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,460 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,460 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,461 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:15:13,462 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,462 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 22c66e6703cd1ab865c50ee7fd3f8725; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60153584, jitterRate=-0.10364174842834473}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:15:13,462 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,463 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 22c66e6703cd1ab865c50ee7fd3f8725: Running coprocessor pre-open hook at 1733235313451Writing region info on filesystem at 1733235313451Initializing all the Stores at 1733235313452 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235313452Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235313452Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235313452Cleaning up temporary data from old regions at 1733235313460 (+8 ms)Running coprocessor post-open hooks at 1733235313462 (+2 ms)Region opened successfully at 1733235313462 2024-12-03T14:15:13,463 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., pid=92, masterSystemTime=1733235313448 2024-12-03T14:15:13,465 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,465 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=OPEN, openSeqNum=5, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:13,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-03T14:15:13,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 in 172 msec 2024-12-03T14:15:13,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T14:15:13,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, REOPEN/MOVE in 494 msec 2024-12-03T14:15:13,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-03T14:15:13,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-12-03T14:15:13,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-12-03T14:15:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T14:15:13,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:13,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,478 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69122fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd48fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54eded6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c73b247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a55cc19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,487 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514e7e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5daf1ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,490 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d619a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40102b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c88e134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,494 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4eb805, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7355d435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,497 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4edc00d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,499 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4c91ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,500 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479ec3ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,502 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496d9c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cebd6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320e3c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:13,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:13,508 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e634ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:13,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:13,510 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab77ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:13,511 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,511 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,511 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,511 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,511 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,511 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:13,512 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,513 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T14:15:13,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,515 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,516 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,517 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:13,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:13,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:15:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:13,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:13,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f5296c9bdb649a29f4abb010b0e20b7_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235313522/Put/seqid=0 2024-12-03T14:15:13,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235373547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235313539, max timestamp=1733235313550Num uniques 859; 2024-12-03T14:15:13,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:13,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235313539, max timestamp=1733235313550Num uniques 859; 2024-12-03T14:15:13,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:13,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235313539, max timestamp=1733235313550Num uniques 859; 2024-12-03T14:15:13,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:13,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742175_1351 (size=17034) 2024-12-03T14:15:13,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235373574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235373575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235373577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235373578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:13,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235373656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:13,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:13,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235373686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235373687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235373696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235373696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235373866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235373896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235373896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235373906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235373906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:13,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:13,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031f5296c9bdb649a29f4abb010b0e20b7_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f5296c9bdb649a29f4abb010b0e20b7_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:13,964 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/cd3122c61f254ce1baabade30816e7de, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:13,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/cd3122c61f254ce1baabade30816e7de is 175, key is test_row_0/A:col10/1733235313522/Put/seqid=0 2024-12-03T14:15:13,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742176_1352 (size=48139) 2024-12-03T14:15:13,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:13,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235374176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235374206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235374216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235374216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235374216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/cd3122c61f254ce1baabade30816e7de 2024-12-03T14:15:14,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/a8fafe4c8d074bb380d1f1037bdcc0b4 is 50, key is test_row_0/B:col10/1733235313522/Put/seqid=0 2024-12-03T14:15:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742177_1353 (size=12001) 2024-12-03T14:15:14,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:14,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:14,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/a8fafe4c8d074bb380d1f1037bdcc0b4 2024-12-03T14:15:14,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/c04c7b0d5474411fa9eeeb3f76a91eca is 50, key is test_row_0/C:col10/1733235313522/Put/seqid=0 2024-12-03T14:15:14,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742178_1354 (size=12001) 2024-12-03T14:15:14,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/c04c7b0d5474411fa9eeeb3f76a91eca 2024-12-03T14:15:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/cd3122c61f254ce1baabade30816e7de as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de 2024-12-03T14:15:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de, entries=250, sequenceid=15, filesize=47.0 K 2024-12-03T14:15:14,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/a8fafe4c8d074bb380d1f1037bdcc0b4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4 2024-12-03T14:15:14,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T14:15:14,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/c04c7b0d5474411fa9eeeb3f76a91eca as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca 2024-12-03T14:15:14,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T14:15:14,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 22c66e6703cd1ab865c50ee7fd3f8725 in 965ms, sequenceid=15, compaction requested=false 2024-12-03T14:15:14,492 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T14:15:14,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:14,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T14:15:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:14,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e198ffb09f654e73898ee3d7f196c8f2_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235313571/Put/seqid=0 2024-12-03T14:15:14,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742179_1355 (size=12154) 2024-12-03T14:15:14,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:14,623 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e198ffb09f654e73898ee3d7f196c8f2_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e198ffb09f654e73898ee3d7f196c8f2_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:14,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/561860645f524b87b598a8032e5038cb, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/561860645f524b87b598a8032e5038cb is 175, key is test_row_0/A:col10/1733235313571/Put/seqid=0 2024-12-03T14:15:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742180_1356 (size=30955) 2024-12-03T14:15:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:14,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235374724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235374726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235374726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235374726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235374733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235374836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235374845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,039 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/561860645f524b87b598a8032e5038cb 2024-12-03T14:15:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/964885897d33463d919244502a1f3b92 is 50, key is test_row_0/B:col10/1733235313571/Put/seqid=0 2024-12-03T14:15:15,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742181_1357 (size=12001) 2024-12-03T14:15:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235375056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235375056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,138 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:15:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235375366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235375366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,451 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/964885897d33463d919244502a1f3b92 2024-12-03T14:15:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/fbf898e0fe584c139335fb7ad35ff2e3 is 50, key is test_row_0/C:col10/1733235313571/Put/seqid=0 2024-12-03T14:15:15,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742182_1358 (size=12001) 2024-12-03T14:15:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235375736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235375738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,744 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,744 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235375746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,753 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/fbf898e0fe584c139335fb7ad35ff2e3 2024-12-03T14:15:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/561860645f524b87b598a8032e5038cb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb 2024-12-03T14:15:15,874 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb, entries=150, sequenceid=48, filesize=30.2 K 2024-12-03T14:15:15,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/964885897d33463d919244502a1f3b92 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92 2024-12-03T14:15:15,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T14:15:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/fbf898e0fe584c139335fb7ad35ff2e3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3 2024-12-03T14:15:15,881 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T14:15:15,882 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1295ms, sequenceid=48, compaction requested=false 2024-12-03T14:15:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T14:15:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T14:15:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:15,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:15:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:15,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:15,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T14:15:15,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3650 sec 2024-12-03T14:15:15,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3740 sec 2024-12-03T14:15:15,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038be5b91551e6456dab81ae0db3b101aa_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:15,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742183_1359 (size=14594) 2024-12-03T14:15:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235375933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235375934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235376036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235376046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235376246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235376266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038be5b91551e6456dab81ae0db3b101aa_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038be5b91551e6456dab81ae0db3b101aa_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:16,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/ffac9e483af54f66b8d7c26934f8d61d, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:16,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/ffac9e483af54f66b8d7c26934f8d61d is 175, key is test_row_0/A:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:16,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742184_1360 (size=39549) 2024-12-03T14:15:16,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235376556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235376586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:16,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/ffac9e483af54f66b8d7c26934f8d61d 2024-12-03T14:15:16,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/743b32bef6d6439b91e4e689efe9f1dd is 50, key is test_row_0/B:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:16,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742185_1361 (size=12001) 2024-12-03T14:15:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235377066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235377096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:17,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/743b32bef6d6439b91e4e689efe9f1dd 2024-12-03T14:15:17,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1b9c0f07d2dc462cb6ecc8c400bb2439 is 50, key is test_row_0/C:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:17,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742186_1362 (size=12001) 2024-12-03T14:15:17,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1b9c0f07d2dc462cb6ecc8c400bb2439 2024-12-03T14:15:17,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/ffac9e483af54f66b8d7c26934f8d61d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d 2024-12-03T14:15:17,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d, entries=200, sequenceid=60, filesize=38.6 K 2024-12-03T14:15:17,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/743b32bef6d6439b91e4e689efe9f1dd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd 2024-12-03T14:15:17,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T14:15:17,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1b9c0f07d2dc462cb6ecc8c400bb2439 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439 2024-12-03T14:15:17,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T14:15:17,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1664ms, sequenceid=60, compaction requested=true 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:17,548 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:17,548 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:17,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:17,549 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:17,549 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:17,549 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/B is initiating minor compaction (all files) 2024-12-03T14:15:17,549 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/A is initiating minor compaction (all files) 2024-12-03T14:15:17,549 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/B in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,549 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/A in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,549 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=35.2 K 2024-12-03T14:15:17,549 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=115.9 K 2024-12-03T14:15:17,549 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,549 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d] 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fafe4c8d074bb380d1f1037bdcc0b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235313522 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3122c61f254ce1baabade30816e7de, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235313520 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 964885897d33463d919244502a1f3b92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733235313571 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 561860645f524b87b598a8032e5038cb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733235313571 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 743b32bef6d6439b91e4e689efe9f1dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314723 2024-12-03T14:15:17,550 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffac9e483af54f66b8d7c26934f8d61d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314700 2024-12-03T14:15:17,558 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:17,559 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d0d12911172f4a838df19bc2a87efd3d_22c66e6703cd1ab865c50ee7fd3f8725 store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:17,561 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d0d12911172f4a838df19bc2a87efd3d_22c66e6703cd1ab865c50ee7fd3f8725, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:17,561 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d0d12911172f4a838df19bc2a87efd3d_22c66e6703cd1ab865c50ee7fd3f8725 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:17,562 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:17,562 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/d30a7a071ac74c4e8334e219b4e28caf is 50, key is test_row_0/B:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:17,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742187_1363 (size=4469) 2024-12-03T14:15:17,566 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#A#compaction#306 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:17,567 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/b0c24443c08e4fc68e5933004fb8a8fd is 175, key is test_row_0/A:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:17,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742188_1364 (size=12104) 2024-12-03T14:15:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742189_1365 (size=31058) 2024-12-03T14:15:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T14:15:17,687 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:17,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T14:15:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:17,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:17,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:17,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:17,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T14:15:17,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235315924, max timestamp=1733235317759Num uniques 859; 2024-12-03T14:15:17,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:17,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235315924, max timestamp=1733235317759Num uniques 859; 2024-12-03T14:15:17,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235315924, max timestamp=1733235317759Num uniques 859; 2024-12-03T14:15:17,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:17,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:17,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:17,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120366b49284d40749d0a39d237b6be28434_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235315933/Put/seqid=0 2024-12-03T14:15:17,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742190_1366 (size=12154) 2024-12-03T14:15:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:17,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235317794, max timestamp=1733235317824Num uniques 859; 2024-12-03T14:15:17,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235317794, max timestamp=1733235317824Num uniques 859; 2024-12-03T14:15:17,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:17,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:17,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235317794, max timestamp=1733235317824Num uniques 859; 2024-12-03T14:15:17,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:17,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:17,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:17,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235317883, max timestamp=1733235317900Num uniques 1068; 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235317883, max timestamp=1733235317900Num uniques 1068; 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235317883, max timestamp=1733235317900Num uniques 1068; 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T14:15:17,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T14:15:17,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235317940, max timestamp=1733235317957Num uniques 1077; 2024-12-03T14:15:17,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235317940, max timestamp=1733235317957Num uniques 1077; 2024-12-03T14:15:17,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:17,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:17,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235317940, max timestamp=1733235317957Num uniques 1077; 2024-12-03T14:15:17,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:17,983 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/d30a7a071ac74c4e8334e219b4e28caf as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d30a7a071ac74c4e8334e219b4e28caf 2024-12-03T14:15:17,984 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/b0c24443c08e4fc68e5933004fb8a8fd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd 2024-12-03T14:15:17,991 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/B of 22c66e6703cd1ab865c50ee7fd3f8725 into d30a7a071ac74c4e8334e219b4e28caf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:17,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:17,991 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/B, priority=13, startTime=1733235317548; duration=0sec 2024-12-03T14:15:17,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:17,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:B 2024-12-03T14:15:17,991 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:17,992 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/A of 22c66e6703cd1ab865c50ee7fd3f8725 into b0c24443c08e4fc68e5933004fb8a8fd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:17,992 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:17,992 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/A, priority=13, startTime=1733235317548; duration=0sec 2024-12-03T14:15:17,992 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:17,993 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/C is initiating minor compaction (all files) 2024-12-03T14:15:17,993 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:17,993 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:A 2024-12-03T14:15:17,993 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/C in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,993 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=35.2 K 2024-12-03T14:15:17,993 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c04c7b0d5474411fa9eeeb3f76a91eca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235313522 2024-12-03T14:15:17,994 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf898e0fe584c139335fb7ad35ff2e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733235313571 2024-12-03T14:15:17,994 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9c0f07d2dc462cb6ecc8c400bb2439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314723 2024-12-03T14:15:17,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:17,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:17,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:17,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:17,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:17,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:18,007 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#C#compaction#309 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:18,008 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/00bc4c968ae04f85a085f9589f415efd is 50, key is test_row_0/C:col10/1733235314723/Put/seqid=0 2024-12-03T14:15:18,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742191_1367 (size=12104) 2024-12-03T14:15:18,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235318017, max timestamp=1733235318032Num uniques 1086; 2024-12-03T14:15:18,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:18,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235318017, max timestamp=1733235318032Num uniques 1086; 2024-12-03T14:15:18,036 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:18,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235318017, max timestamp=1733235318032Num uniques 1086; 2024-12-03T14:15:18,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:18,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733235318063, max timestamp=1733235318086Num uniques 1095; 2024-12-03T14:15:18,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T14:15:18,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733235318063, max timestamp=1733235318086Num uniques 1095; 2024-12-03T14:15:18,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T14:15:18,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733235318063, max timestamp=1733235318086Num uniques 1095; 2024-12-03T14:15:18,091 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T14:15:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235378140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235378141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733235318122, max timestamp=1733235318148Num uniques 1104; 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733235318122, max timestamp=1733235318148Num uniques 1104; 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733235318122, max timestamp=1733235318148Num uniques 1104; 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-03T14:15:18,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:18,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120366b49284d40749d0a39d237b6be28434_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120366b49284d40749d0a39d237b6be28434_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:18,176 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/758aed63ebe84e7a81ceae475bda8323, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:18,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/758aed63ebe84e7a81ceae475bda8323 is 175, key is test_row_0/A:col10/1733235315933/Put/seqid=0 2024-12-03T14:15:18,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742192_1368 (size=30955) 2024-12-03T14:15:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235378207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733235318160, max timestamp=1733235318214Num uniques 1113; 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733235318160, max timestamp=1733235318214Num uniques 1113; 2024-12-03T14:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733235318160, max timestamp=1733235318214Num uniques 1113; 2024-12-03T14:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-03T14:15:18,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-03T14:15:18,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-03T14:15:18,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733235318239, max timestamp=1733235318265Num uniques 1022; 2024-12-03T14:15:18,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-03T14:15:18,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733235318239, max timestamp=1733235318265Num uniques 1022; 2024-12-03T14:15:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-03T14:15:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1733235318239, max timestamp=1733235318265Num uniques 1022; 2024-12-03T14:15:18,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-12-03T14:15:18,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235378294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235378295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733235318290, max timestamp=1733235318303Num uniques 1181; 2024-12-03T14:15:18,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-03T14:15:18,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733235318290, max timestamp=1733235318303Num uniques 1181; 2024-12-03T14:15:18,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-03T14:15:18,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733235318290, max timestamp=1733235318303Num uniques 1181; 2024-12-03T14:15:18,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=10576, suffix heap size=150792,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=231 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733235318334, max timestamp=1733235318363Num uniques 1090; 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733235318334, max timestamp=1733235318363Num uniques 1090; 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733235318334, max timestamp=1733235318363Num uniques 1090; 2024-12-03T14:15:18,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T14:15:18,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733235318399, max timestamp=1733235318419Num uniques 1149; 2024-12-03T14:15:18,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-12-03T14:15:18,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733235318399, max timestamp=1733235318419Num uniques 1149; 2024-12-03T14:15:18,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-12-03T14:15:18,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1733235318399, max timestamp=1733235318419Num uniques 1149; 2024-12-03T14:15:18,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-12-03T14:15:18,432 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/00bc4c968ae04f85a085f9589f415efd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/00bc4c968ae04f85a085f9589f415efd 2024-12-03T14:15:18,438 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/C of 22c66e6703cd1ab865c50ee7fd3f8725 into 00bc4c968ae04f85a085f9589f415efd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:18,438 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:18,438 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/C, priority=13, startTime=1733235317548; duration=0sec 2024-12-03T14:15:18,438 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:18,438 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:C 2024-12-03T14:15:18,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733235318455, max timestamp=1733235318468Num uniques 1158; 2024-12-03T14:15:18,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-12-03T14:15:18,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733235318455, max timestamp=1733235318468Num uniques 1158; 2024-12-03T14:15:18,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-12-03T14:15:18,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733235318455, max timestamp=1733235318468Num uniques 1158; 2024-12-03T14:15:18,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-12-03T14:15:18,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733235318503, max timestamp=1733235318558Num uniques 1067; 2024-12-03T14:15:18,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733235318503, max timestamp=1733235318558Num uniques 1067; 2024-12-03T14:15:18,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-03T14:15:18,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733235318503, max timestamp=1733235318558Num uniques 1067; 2024-12-03T14:15:18,561 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-03T14:15:18,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-03T14:15:18,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/758aed63ebe84e7a81ceae475bda8323 2024-12-03T14:15:18,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/62323db681f44075b1d08072ab963ce6 is 50, key is test_row_0/B:col10/1733235315933/Put/seqid=0 2024-12-03T14:15:18,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742193_1369 (size=12001) 2024-12-03T14:15:18,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/62323db681f44075b1d08072ab963ce6 2024-12-03T14:15:18,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733235318586, max timestamp=1733235318616Num uniques 1026; 2024-12-03T14:15:18,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfbb87c14b4145d5b6445115e29605ec is 50, key is test_row_0/C:col10/1733235315933/Put/seqid=0 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733235318586, max timestamp=1733235318616Num uniques 1026; 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733235318586, max timestamp=1733235318616Num uniques 1026; 2024-12-03T14:15:18,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-12-03T14:15:18,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742194_1370 (size=12001) 2024-12-03T14:15:18,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfbb87c14b4145d5b6445115e29605ec 2024-12-03T14:15:18,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/758aed63ebe84e7a81ceae475bda8323 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323 2024-12-03T14:15:18,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323, entries=150, sequenceid=82, filesize=30.2 K 2024-12-03T14:15:18,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/62323db681f44075b1d08072ab963ce6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6 2024-12-03T14:15:18,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T14:15:18,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfbb87c14b4145d5b6445115e29605ec as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec 2024-12-03T14:15:18,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733235318648, max timestamp=1733235318662Num uniques 1035; 2024-12-03T14:15:18,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-03T14:15:18,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733235318648, max timestamp=1733235318662Num uniques 1035; 2024-12-03T14:15:18,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-03T14:15:18,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1733235318648, max timestamp=1733235318662Num uniques 1035; 2024-12-03T14:15:18,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-12-03T14:15:18,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T14:15:18,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=46.96 KB/48090 for 22c66e6703cd1ab865c50ee7fd3f8725 in 915ms, sequenceid=82, compaction requested=false 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:18,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=53.67 KB heapSize=138.80 KB 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:18,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f29ff617b3ae4433aac04b7e43dea374_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:18,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742195_1371 (size=22618) 2024-12-03T14:15:18,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:18,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f29ff617b3ae4433aac04b7e43dea374_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f29ff617b3ae4433aac04b7e43dea374_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:18,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/20ae75530dc541c3a9478727addc178f, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:18,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/20ae75530dc541c3a9478727addc178f is 175, key is test_row_0/A:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:18,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742196_1372 (size=66023) 2024-12-03T14:15:18,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235318714, max timestamp=1733235318731Num uniques 859; 2024-12-03T14:15:18,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235318714, max timestamp=1733235318731Num uniques 859; 2024-12-03T14:15:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235318714, max timestamp=1733235318731Num uniques 859; 2024-12-03T14:15:18,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:18,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235378773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235378775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235378776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235378777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235378778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235378885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235378896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235378896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235378896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235378896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:18,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:18,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235379095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235379106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235379106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235379106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235379106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/20ae75530dc541c3a9478727addc178f 2024-12-03T14:15:19,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/4a947b2100994cfa9b31964fca148e63 is 50, key is test_row_0/B:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742197_1373 (size=12301) 2024-12-03T14:15:19,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235379406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235379416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235379416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235379416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235379416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/4a947b2100994cfa9b31964fca148e63 2024-12-03T14:15:19,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a7f95b6f6f4d4d099bf35f5e3f59952c is 50, key is test_row_0/C:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:19,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742198_1374 (size=12301) 2024-12-03T14:15:19,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:19,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235379919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235379925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235379926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235379926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235379926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:19,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a7f95b6f6f4d4d099bf35f5e3f59952c 2024-12-03T14:15:19,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/20ae75530dc541c3a9478727addc178f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f 2024-12-03T14:15:19,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f, entries=350, sequenceid=349, filesize=64.5 K 2024-12-03T14:15:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/4a947b2100994cfa9b31964fca148e63 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63 2024-12-03T14:15:19,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63, entries=150, sequenceid=349, filesize=12.0 K 2024-12-03T14:15:19,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a7f95b6f6f4d4d099bf35f5e3f59952c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c 2024-12-03T14:15:19,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c, entries=150, sequenceid=349, filesize=12.0 K 2024-12-03T14:15:19,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~138.75 KB/142080, currentSize=134.18 KB/137400 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1291ms, sequenceid=349, compaction requested=true 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:19,968 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:19,968 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:19,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128036 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/B is initiating minor compaction (all files) 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/A is initiating minor compaction (all files) 2024-12-03T14:15:19,969 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/B in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,969 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/A in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,969 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d30a7a071ac74c4e8334e219b4e28caf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=35.6 K 2024-12-03T14:15:19,969 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=125.0 K 2024-12-03T14:15:19,969 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f] 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d30a7a071ac74c4e8334e219b4e28caf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314723 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c24443c08e4fc68e5933004fb8a8fd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314723 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 62323db681f44075b1d08072ab963ce6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733235315924 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 758aed63ebe84e7a81ceae475bda8323, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733235315924 2024-12-03T14:15:19,969 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a947b2100994cfa9b31964fca148e63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318657 2024-12-03T14:15:19,970 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ae75530dc541c3a9478727addc178f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318651 2024-12-03T14:15:19,976 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:19,978 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:19,978 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/2a8ce41b520143e2af72742426960919 is 50, key is test_row_0/B:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:19,979 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412039fc139d5eb4a47d7bf759e872d31b577_22c66e6703cd1ab865c50ee7fd3f8725 store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:19,981 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412039fc139d5eb4a47d7bf759e872d31b577_22c66e6703cd1ab865c50ee7fd3f8725, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:19,981 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039fc139d5eb4a47d7bf759e872d31b577_22c66e6703cd1ab865c50ee7fd3f8725 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742199_1375 (size=12507) 2024-12-03T14:15:19,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:19,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:19,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:19,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742200_1376 (size=4469) 2024-12-03T14:15:19,996 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/2a8ce41b520143e2af72742426960919 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/2a8ce41b520143e2af72742426960919 2024-12-03T14:15:20,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037a097c1cea21443fb47689f6b0fc3c23_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235318774/Put/seqid=0 2024-12-03T14:15:20,005 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/B of 22c66e6703cd1ab865c50ee7fd3f8725 into 2a8ce41b520143e2af72742426960919(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:20,005 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:20,005 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/B, priority=13, startTime=1733235319968; duration=0sec 2024-12-03T14:15:20,005 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:20,005 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:B 2024-12-03T14:15:20,005 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:20,006 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:20,006 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/C is initiating minor compaction (all files) 2024-12-03T14:15:20,006 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/C in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:20,006 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/00bc4c968ae04f85a085f9589f415efd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=35.6 K 2024-12-03T14:15:20,006 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 00bc4c968ae04f85a085f9589f415efd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733235314723 2024-12-03T14:15:20,007 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting bfbb87c14b4145d5b6445115e29605ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733235315924 2024-12-03T14:15:20,007 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f95b6f6f4d4d099bf35f5e3f59952c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318657 2024-12-03T14:15:20,023 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:20,023 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/12817a3fb48b4976a6a3bf3930632981 is 50, key is test_row_0/C:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:20,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742201_1377 (size=12454) 2024-12-03T14:15:20,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742202_1378 (size=12507) 2024-12-03T14:15:20,395 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:20,396 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/2a105666e45040a78784ba658073baa3 is 175, key is test_row_0/A:col10/1733235318675/Put/seqid=0 2024-12-03T14:15:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742203_1379 (size=31461) 2024-12-03T14:15:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,427 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037a097c1cea21443fb47689f6b0fc3c23_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037a097c1cea21443fb47689f6b0fc3c23_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:20,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fe1bb87bb5b045fc992313b4d6431e86, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:20,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fe1bb87bb5b045fc992313b4d6431e86 is 175, key is test_row_0/A:col10/1733235318774/Put/seqid=0 2024-12-03T14:15:20,439 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/12817a3fb48b4976a6a3bf3930632981 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/12817a3fb48b4976a6a3bf3930632981 2024-12-03T14:15:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742204_1380 (size=31255) 2024-12-03T14:15:20,445 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/C of 22c66e6703cd1ab865c50ee7fd3f8725 into 12817a3fb48b4976a6a3bf3930632981(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:20,445 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:20,445 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/C, priority=13, startTime=1733235319968; duration=0sec 2024-12-03T14:15:20,445 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:20,445 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:C 2024-12-03T14:15:20,805 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/2a105666e45040a78784ba658073baa3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3 2024-12-03T14:15:20,810 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/A of 22c66e6703cd1ab865c50ee7fd3f8725 into 2a105666e45040a78784ba658073baa3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:20,810 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:20,810 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/A, priority=13, startTime=1733235319968; duration=0sec 2024-12-03T14:15:20,810 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:20,810 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:A 2024-12-03T14:15:20,840 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fe1bb87bb5b045fc992313b4d6431e86 2024-12-03T14:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/90bf5bfc4db04667aa9452955e24e2d1 is 50, key is test_row_0/B:col10/1733235318774/Put/seqid=0 2024-12-03T14:15:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742205_1381 (size=12301) 2024-12-03T14:15:20,863 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/90bf5bfc4db04667aa9452955e24e2d1 2024-12-03T14:15:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a9aa62f00a244f3a9eef0aad047c5ed9 is 50, key is test_row_0/C:col10/1733235318774/Put/seqid=0 2024-12-03T14:15:20,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742206_1382 (size=12301) 2024-12-03T14:15:20,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a9aa62f00a244f3a9eef0aad047c5ed9 2024-12-03T14:15:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fe1bb87bb5b045fc992313b4d6431e86 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86 2024-12-03T14:15:20,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86, entries=150, sequenceid=386, filesize=30.5 K 2024-12-03T14:15:20,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/90bf5bfc4db04667aa9452955e24e2d1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1 2024-12-03T14:15:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,891 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1, entries=150, sequenceid=386, filesize=12.0 K 2024-12-03T14:15:20,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a9aa62f00a244f3a9eef0aad047c5ed9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9 2024-12-03T14:15:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9, entries=150, sequenceid=386, filesize=12.0 K 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 22c66e6703cd1ab865c50ee7fd3f8725 in 909ms, sequenceid=386, compaction requested=false 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T14:15:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2080 sec 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.2130 sec 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:21,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa02dd3fbeee445f9d30fcaedd3490f3_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742207_1383 (size=22618) 2024-12-03T14:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235321045, max timestamp=1733235321151Num uniques 859; 2024-12-03T14:15:21,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:15:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235321045, max timestamp=1733235321151Num uniques 859; 2024-12-03T14:15:21,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:15:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733235321045, max timestamp=1733235321151Num uniques 859; 2024-12-03T14:15:21,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T14:15:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235381166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235381174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235381176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235381177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235381179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235381286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235381286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235381297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235381297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235381306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:21,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aa02dd3fbeee445f9d30fcaedd3490f3_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa02dd3fbeee445f9d30fcaedd3490f3_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:21,495 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/c2e0210a4a27493eb2c2ead0f4af6638, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:21,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/c2e0210a4a27493eb2c2ead0f4af6638 is 175, key is test_row_0/A:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235381507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235381507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235381527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235381527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235381527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742208_1384 (size=66023) 2024-12-03T14:15:21,536 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/c2e0210a4a27493eb2c2ead0f4af6638 2024-12-03T14:15:21,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/d38a3e94ae1042f5b8c0f657c58c0c55 is 50, key is test_row_0/B:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742209_1385 (size=12301) 2024-12-03T14:15:21,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/d38a3e94ae1042f5b8c0f657c58c0c55 2024-12-03T14:15:21,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a05e587f3f224fb6ac933b2f31b84241 is 50, key is test_row_0/C:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742210_1386 (size=12301) 2024-12-03T14:15:21,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a05e587f3f224fb6ac933b2f31b84241 2024-12-03T14:15:21,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/c2e0210a4a27493eb2c2ead0f4af6638 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638 2024-12-03T14:15:21,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638, entries=350, sequenceid=399, filesize=64.5 K 2024-12-03T14:15:21,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/d38a3e94ae1042f5b8c0f657c58c0c55 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55 2024-12-03T14:15:21,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55, entries=150, sequenceid=399, filesize=12.0 K 2024-12-03T14:15:21,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/a05e587f3f224fb6ac933b2f31b84241 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241 2024-12-03T14:15:21,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241, entries=150, sequenceid=399, filesize=12.0 K 2024-12-03T14:15:21,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=161.02 KB/164880 for 22c66e6703cd1ab865c50ee7fd3f8725 in 628ms, sequenceid=399, compaction requested=true 2024-12-03T14:15:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:21,673 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:21,673 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:21,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:21,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:21,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:21,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:21,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:21,676 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:21,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/B is initiating minor compaction (all files) 2024-12-03T14:15:21,676 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/A is initiating minor compaction (all files) 2024-12-03T14:15:21,676 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/B in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:21,676 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/A in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:21,676 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=125.7 K 2024-12-03T14:15:21,676 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/2a8ce41b520143e2af72742426960919, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=36.2 K 2024-12-03T14:15:21,676 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:21,676 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638] 2024-12-03T14:15:21,677 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8ce41b520143e2af72742426960919, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318657 2024-12-03T14:15:21,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a105666e45040a78784ba658073baa3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318657 2024-12-03T14:15:21,677 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 90bf5bfc4db04667aa9452955e24e2d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733235318758 2024-12-03T14:15:21,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1bb87bb5b045fc992313b4d6431e86, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733235318758 2024-12-03T14:15:21,677 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d38a3e94ae1042f5b8c0f657c58c0c55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321028 2024-12-03T14:15:21,678 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2e0210a4a27493eb2c2ead0f4af6638, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321019 2024-12-03T14:15:21,698 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:21,699 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73cf838aa59841fda29039de20ae2caa is 50, key is test_row_0/B:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,713 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:21,730 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034199b03a9ffd423fa28fcd3f60b22785_22c66e6703cd1ab865c50ee7fd3f8725 store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:21,732 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034199b03a9ffd423fa28fcd3f60b22785_22c66e6703cd1ab865c50ee7fd3f8725, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:21,732 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034199b03a9ffd423fa28fcd3f60b22785_22c66e6703cd1ab865c50ee7fd3f8725 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742211_1387 (size=12609) 2024-12-03T14:15:21,797 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73cf838aa59841fda29039de20ae2caa as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73cf838aa59841fda29039de20ae2caa 2024-12-03T14:15:21,805 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/B of 22c66e6703cd1ab865c50ee7fd3f8725 into 73cf838aa59841fda29039de20ae2caa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:21,805 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:21,805 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/B, priority=13, startTime=1733235321673; duration=0sec 2024-12-03T14:15:21,805 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:21,805 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:B 2024-12-03T14:15:21,805 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:21,807 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:21,807 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/C is initiating minor compaction (all files) 2024-12-03T14:15:21,807 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/C in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:21,807 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/12817a3fb48b4976a6a3bf3930632981, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=36.2 K 2024-12-03T14:15:21,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 12817a3fb48b4976a6a3bf3930632981, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733235318657 2024-12-03T14:15:21,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a9aa62f00a244f3a9eef0aad047c5ed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733235318758 2024-12-03T14:15:21,810 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a05e587f3f224fb6ac933b2f31b84241, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321028 2024-12-03T14:15:21,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742212_1388 (size=4469) 2024-12-03T14:15:21,818 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#A#compaction#325 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:21,819 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/d286f9643f864901bbe9535ad2730837 is 175, key is test_row_0/A:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=167.72 KB heapSize=418.97 KB 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:21,834 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:21,834 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/9c4d9d131bc540c09642c91f3d8d36e1 is 50, key is test_row_0/C:col10/1733235321028/Put/seqid=0 2024-12-03T14:15:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T14:15:21,857 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:21,857 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T14:15:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:21,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203629f6e8054c245d5a886389d4efbcfb6_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235321174/Put/seqid=0 2024-12-03T14:15:21,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235381849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235381850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235381851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235381852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742213_1389 (size=31563) 2024-12-03T14:15:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235381854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742214_1390 (size=12609) 2024-12-03T14:15:21,918 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/9c4d9d131bc540c09642c91f3d8d36e1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/9c4d9d131bc540c09642c91f3d8d36e1 2024-12-03T14:15:21,926 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/C of 22c66e6703cd1ab865c50ee7fd3f8725 into 9c4d9d131bc540c09642c91f3d8d36e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:21,926 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:21,926 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/C, priority=13, startTime=1733235321676; duration=0sec 2024-12-03T14:15:21,926 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:21,926 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:C 2024-12-03T14:15:21,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742215_1391 (size=14994) 2024-12-03T14:15:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:21,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235381977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235381977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235381977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235381977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235381977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:22,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235382187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235382196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235382196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235382196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235382196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,277 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/d286f9643f864901bbe9535ad2730837 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837 2024-12-03T14:15:22,290 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/A of 22c66e6703cd1ab865c50ee7fd3f8725 into d286f9643f864901bbe9535ad2730837(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:22,290 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:22,290 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/A, priority=13, startTime=1733235321673; duration=0sec 2024-12-03T14:15:22,291 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:22,291 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:A 2024-12-03T14:15:22,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:22,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203629f6e8054c245d5a886389d4efbcfb6_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203629f6e8054c245d5a886389d4efbcfb6_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:22,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/603d097fb4ab40e1befec35cc1d5977c, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:22,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/603d097fb4ab40e1befec35cc1d5977c is 175, key is test_row_0/A:col10/1733235321174/Put/seqid=0 2024-12-03T14:15:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742216_1392 (size=39949) 2024-12-03T14:15:22,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:22,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235382506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235382507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235382507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235382516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235382516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:22,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/603d097fb4ab40e1befec35cc1d5977c 2024-12-03T14:15:22,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0475e3c39bf34379a4e657608b0cb32d is 50, key is test_row_0/B:col10/1733235321174/Put/seqid=0 2024-12-03T14:15:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742217_1393 (size=12301) 2024-12-03T14:15:22,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:22,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:23,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235383025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235383026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235383026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235383026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235383036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:23,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0475e3c39bf34379a4e657608b0cb32d 2024-12-03T14:15:23,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/cd8c7075187b48b99dcaccdbe262b3ac is 50, key is test_row_0/C:col10/1733235321174/Put/seqid=0 2024-12-03T14:15:23,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742218_1394 (size=12301) 2024-12-03T14:15:23,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:23,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/cd8c7075187b48b99dcaccdbe262b3ac 2024-12-03T14:15:23,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/603d097fb4ab40e1befec35cc1d5977c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c 2024-12-03T14:15:23,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c, entries=200, sequenceid=428, filesize=39.0 K 2024-12-03T14:15:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0475e3c39bf34379a4e657608b0cb32d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d 2024-12-03T14:15:23,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d, entries=150, sequenceid=428, filesize=12.0 K 2024-12-03T14:15:23,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/cd8c7075187b48b99dcaccdbe262b3ac as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac 2024-12-03T14:15:23,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac, entries=150, sequenceid=428, filesize=12.0 K 2024-12-03T14:15:23,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~418.92 KB/428976, currentSize=40.25 KB/41220 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1741ms, sequenceid=428, compaction requested=false 2024-12-03T14:15:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:23,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:23,710 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:23,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b068ea7aed8147ffa7312d4b005a4b3e_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235321849/Put/seqid=0 2024-12-03T14:15:23,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742219_1395 (size=12454) 2024-12-03T14:15:24,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:24,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235324061, max timestamp=1733235324075Num uniques 859; 2024-12-03T14:15:24,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:24,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235324061, max timestamp=1733235324075Num uniques 859; 2024-12-03T14:15:24,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:24,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235324061, max timestamp=1733235324075Num uniques 859; 2024-12-03T14:15:24,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:24,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235384098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235384100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235384101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235384103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235384104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:24,121 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b068ea7aed8147ffa7312d4b005a4b3e_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b068ea7aed8147ffa7312d4b005a4b3e_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:24,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/9daae805286b4b199845e8b53ebced70, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:24,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/9daae805286b4b199845e8b53ebced70 is 175, key is test_row_0/A:col10/1733235321849/Put/seqid=0 2024-12-03T14:15:24,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742220_1396 (size=31255) 2024-12-03T14:15:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235384216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235384216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235384216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235384216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235384226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235384426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235384426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235384426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235384426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235384436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,525 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/9daae805286b4b199845e8b53ebced70 2024-12-03T14:15:24,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/9e48d3b352684d20b4f8d9a8000b737a is 50, key is test_row_0/B:col10/1733235321849/Put/seqid=0 2024-12-03T14:15:24,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742221_1397 (size=12301) 2024-12-03T14:15:24,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235384736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235384736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235384737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235384737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235384746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:24,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:24,934 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/9e48d3b352684d20b4f8d9a8000b737a 2024-12-03T14:15:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/50ff93e400cd433199e16c250ceb6522 is 50, key is test_row_0/C:col10/1733235321849/Put/seqid=0 2024-12-03T14:15:24,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742222_1398 (size=12301) 2024-12-03T14:15:25,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235385246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235385256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235385256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235385256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:25,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235385266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:25,343 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/50ff93e400cd433199e16c250ceb6522 2024-12-03T14:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/9daae805286b4b199845e8b53ebced70 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70 2024-12-03T14:15:25,349 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70, entries=150, sequenceid=439, filesize=30.5 K 2024-12-03T14:15:25,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/9e48d3b352684d20b4f8d9a8000b737a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a 2024-12-03T14:15:25,353 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T14:15:25,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/50ff93e400cd433199e16c250ceb6522 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522 2024-12-03T14:15:25,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522, entries=150, sequenceid=439, filesize=12.0 K 2024-12-03T14:15:25,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1647ms, sequenceid=439, compaction requested=true 2024-12-03T14:15:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T14:15:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T14:15:25,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T14:15:25,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4940 sec 2024-12-03T14:15:25,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.5030 sec 2024-12-03T14:15:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T14:15:26,026 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:26,026 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T14:15:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T14:15:26,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:26,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:26,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T14:15:26,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:26,181 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:26,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b719ff75724944359936148a1cd1e316_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235324102/Put/seqid=0 2024-12-03T14:15:26,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742223_1399 (size=12454) 2024-12-03T14:15:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:26,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235386276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235386276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235386279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,282 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:26,282 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235386281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235386282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T14:15:26,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235386386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235386396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235386396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:26,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b719ff75724944359936148a1cd1e316_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b719ff75724944359936148a1cd1e316_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:26,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/eddf5bea82b84d9a8e720ce82888227a, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:26,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/eddf5bea82b84d9a8e720ce82888227a is 175, key is test_row_0/A:col10/1733235324102/Put/seqid=0 2024-12-03T14:15:26,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235386606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235386606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235386606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742224_1400 (size=31255) 2024-12-03T14:15:26,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=476, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/eddf5bea82b84d9a8e720ce82888227a 2024-12-03T14:15:26,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/e3108a0c08e14698886babb28bb7d80c is 50, key is test_row_0/B:col10/1733235324102/Put/seqid=0 2024-12-03T14:15:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T14:15:26,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742225_1401 (size=12301) 2024-12-03T14:15:26,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235386926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235386926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235386926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:26,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,072 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/e3108a0c08e14698886babb28bb7d80c 2024-12-03T14:15:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1629d508b8674249b6937c95f515cfe8 is 50, key is test_row_0/C:col10/1733235324102/Put/seqid=0 2024-12-03T14:15:27,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742226_1402 (size=12301) 2024-12-03T14:15:27,092 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1629d508b8674249b6937c95f515cfe8 2024-12-03T14:15:27,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/eddf5bea82b84d9a8e720ce82888227a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a 2024-12-03T14:15:27,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a, entries=150, sequenceid=476, filesize=30.5 K 2024-12-03T14:15:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/e3108a0c08e14698886babb28bb7d80c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c 2024-12-03T14:15:27,108 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c, entries=150, sequenceid=476, filesize=12.0 K 2024-12-03T14:15:27,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/1629d508b8674249b6937c95f515cfe8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8 2024-12-03T14:15:27,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8, entries=150, sequenceid=476, filesize=12.0 K 2024-12-03T14:15:27,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 22c66e6703cd1ab865c50ee7fd3f8725 in 931ms, sequenceid=476, compaction requested=true 2024-12-03T14:15:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T14:15:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T14:15:27,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T14:15:27,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-12-03T14:15:27,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0880 sec 2024-12-03T14:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T14:15:27,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:27,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T14:15:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:27,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:27,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:27,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:27,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:27,326 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120330cdae902bfa4c059d95d9b806c50bcf_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235326281/Put/seqid=0 2024-12-03T14:15:27,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742227_1403 (size=12454) 2024-12-03T14:15:27,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:27,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120330cdae902bfa4c059d95d9b806c50bcf_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330cdae902bfa4c059d95d9b806c50bcf_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:27,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e7ec547df4254212bcb3339787fe76fb, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e7ec547df4254212bcb3339787fe76fb is 175, key is test_row_0/A:col10/1733235326281/Put/seqid=0 2024-12-03T14:15:27,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742228_1404 (size=31255) 2024-12-03T14:15:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:27,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:27,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235327490, max timestamp=1733235327522Num uniques 859; 2024-12-03T14:15:27,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235327490, max timestamp=1733235327522Num uniques 859; 2024-12-03T14:15:27,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:27,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:27,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235327490, max timestamp=1733235327522Num uniques 859; 2024-12-03T14:15:27,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:15:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235387670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235387672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235387680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,789 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=485, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e7ec547df4254212bcb3339787fe76fb 2024-12-03T14:15:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:27,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73fe96cd4e224924a278787fcbaa7c65 is 50, key is test_row_0/B:col10/1733235326281/Put/seqid=0 2024-12-03T14:15:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235387786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235387786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742229_1405 (size=12301) 2024-12-03T14:15:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235387806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235388006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235388006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235388026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73fe96cd4e224924a278787fcbaa7c65 2024-12-03T14:15:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/d076b2b037504a91a07392420f2e7521 is 50, key is test_row_0/C:col10/1733235326281/Put/seqid=0 2024-12-03T14:15:28,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742230_1406 (size=12301) 2024-12-03T14:15:28,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235388296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,301 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235388306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,311 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235388326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235388326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235388346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/d076b2b037504a91a07392420f2e7521 2024-12-03T14:15:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e7ec547df4254212bcb3339787fe76fb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb 2024-12-03T14:15:28,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb, entries=150, sequenceid=485, filesize=30.5 K 2024-12-03T14:15:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73fe96cd4e224924a278787fcbaa7c65 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65 2024-12-03T14:15:28,624 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65, entries=150, sequenceid=485, filesize=12.0 K 2024-12-03T14:15:28,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/d076b2b037504a91a07392420f2e7521 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521 2024-12-03T14:15:28,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521, entries=150, sequenceid=485, filesize=12.0 K 2024-12-03T14:15:28,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1305ms, sequenceid=485, compaction requested=true 2024-12-03T14:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T14:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T14:15:28,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T14:15:28,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-12-03T14:15:28,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4680 sec 2024-12-03T14:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:28,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-03T14:15:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:15:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:15:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:28,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T14:15:28,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120360af6054ee8a45369e9d0a765fcea93d_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742231_1407 (size=14994) 2024-12-03T14:15:28,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:28,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120360af6054ee8a45369e9d0a765fcea93d_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360af6054ee8a45369e9d0a765fcea93d_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:28,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/8ca7250e5b3b4d5ba3367f1a92ea9e26, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:28,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 is 175, key is test_row_0/A:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:28,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742232_1408 (size=39949) 2024-12-03T14:15:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235388856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235388857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235388860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235388976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235388977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235389196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235389197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=523, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 2024-12-03T14:15:29,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0164b345ea50475dbe3ecf016a6b34b2 is 50, key is test_row_0/B:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:29,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742233_1409 (size=12301) 2024-12-03T14:15:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T14:15:29,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:29,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T14:15:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:29,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:29,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:29,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:29,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235389516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235389516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:29,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:29,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0164b345ea50475dbe3ecf016a6b34b2 2024-12-03T14:15:29,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfc6014315d748e9b6c5dcfc4c409150 is 50, key is test_row_0/C:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742234_1410 (size=12301) 2024-12-03T14:15:29,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235389886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:29,889 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:29,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:29,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235390026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235390026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:30,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfc6014315d748e9b6c5dcfc4c409150 2024-12-03T14:15:30,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 2024-12-03T14:15:30,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26, entries=200, sequenceid=523, filesize=39.0 K 2024-12-03T14:15:30,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/0164b345ea50475dbe3ecf016a6b34b2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2 2024-12-03T14:15:30,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:30,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:30,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2, entries=150, sequenceid=523, filesize=12.0 K 2024-12-03T14:15:30,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/bfc6014315d748e9b6c5dcfc4c409150 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150 2024-12-03T14:15:30,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150, entries=150, sequenceid=523, filesize=12.0 K 2024-12-03T14:15:30,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1249ms, sequenceid=523, compaction requested=true 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:30,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:30,089 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T14:15:30,089 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T14:15:30,090 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74114 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T14:15:30,090 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 205226 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T14:15:30,090 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/A is initiating minor compaction (all files) 2024-12-03T14:15:30,090 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/B is initiating minor compaction (all files) 2024-12-03T14:15:30,090 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/A in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,090 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/B in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,091 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73cf838aa59841fda29039de20ae2caa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=72.4 K 2024-12-03T14:15:30,091 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=200.4 K 2024-12-03T14:15:30,091 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26] 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 73cf838aa59841fda29039de20ae2caa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321028 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d286f9643f864901bbe9535ad2730837, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321028 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0475e3c39bf34379a4e657608b0cb32d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235321174 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603d097fb4ab40e1befec35cc1d5977c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235321164 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e48d3b352684d20b4f8d9a8000b737a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733235321847 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9daae805286b4b199845e8b53ebced70, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733235321847 2024-12-03T14:15:30,091 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e3108a0c08e14698886babb28bb7d80c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733235324100 2024-12-03T14:15:30,092 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting eddf5bea82b84d9a8e720ce82888227a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733235324100 2024-12-03T14:15:30,092 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fe96cd4e224924a278787fcbaa7c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733235326261 2024-12-03T14:15:30,092 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ec547df4254212bcb3339787fe76fb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733235326261 2024-12-03T14:15:30,092 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0164b345ea50475dbe3ecf016a6b34b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:30,092 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ca7250e5b3b4d5ba3367f1a92ea9e26, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:30,098 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:30,101 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412031695962aa73f4a7cb6e3b807a5133e64_22c66e6703cd1ab865c50ee7fd3f8725 store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:30,101 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:30,101 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/1c00d668dad946ef9f0ced567476a7e0 is 50, key is test_row_0/B:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:30,104 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412031695962aa73f4a7cb6e3b807a5133e64_22c66e6703cd1ab865c50ee7fd3f8725, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:30,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742235_1411 (size=12813) 2024-12-03T14:15:30,104 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031695962aa73f4a7cb6e3b807a5133e64_22c66e6703cd1ab865c50ee7fd3f8725 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:30,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742236_1412 (size=4469) 2024-12-03T14:15:30,108 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#A#compaction#343 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:30,108 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fb2e34591d5b4d7890eda54e111bafee is 175, key is test_row_0/A:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:30,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742237_1413 (size=31767) 2024-12-03T14:15:30,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T14:15:30,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,235 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:30,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ac6a966738d44655a7106c06ce7906ae_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235328855/Put/seqid=0 2024-12-03T14:15:30,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742238_1414 (size=12454) 2024-12-03T14:15:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:30,507 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/1c00d668dad946ef9f0ced567476a7e0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/1c00d668dad946ef9f0ced567476a7e0 2024-12-03T14:15:30,512 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/B of 22c66e6703cd1ab865c50ee7fd3f8725 into 1c00d668dad946ef9f0ced567476a7e0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:30,512 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:30,512 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/B, priority=10, startTime=1733235330089; duration=0sec 2024-12-03T14:15:30,512 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:30,512 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:B 2024-12-03T14:15:30,512 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T14:15:30,514 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74114 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T14:15:30,514 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/C is initiating minor compaction (all files) 2024-12-03T14:15:30,514 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/C in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:30,514 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/9c4d9d131bc540c09642c91f3d8d36e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=72.4 K 2024-12-03T14:15:30,514 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4d9d131bc540c09642c91f3d8d36e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733235321028 2024-12-03T14:15:30,514 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8c7075187b48b99dcaccdbe262b3ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235321174 2024-12-03T14:15:30,515 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ff93e400cd433199e16c250ceb6522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733235321847 2024-12-03T14:15:30,515 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/fb2e34591d5b4d7890eda54e111bafee as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee 2024-12-03T14:15:30,515 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1629d508b8674249b6937c95f515cfe8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733235324100 2024-12-03T14:15:30,515 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d076b2b037504a91a07392420f2e7521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1733235326261 2024-12-03T14:15:30,516 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc6014315d748e9b6c5dcfc4c409150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:30,519 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/A of 22c66e6703cd1ab865c50ee7fd3f8725 into fb2e34591d5b4d7890eda54e111bafee(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:30,520 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:30,520 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/A, priority=10, startTime=1733235330089; duration=0sec 2024-12-03T14:15:30,520 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:30,520 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:A 2024-12-03T14:15:30,525 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:30,525 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/2fbcf6377a84442abd0666b420c414cc is 50, key is test_row_0/C:col10/1733235328839/Put/seqid=0 2024-12-03T14:15:30,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742239_1415 (size=12813) 2024-12-03T14:15:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:30,646 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ac6a966738d44655a7106c06ce7906ae_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ac6a966738d44655a7106c06ce7906ae_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:30,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/bff1d427ac154d0e9cc98830785f1160, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:30,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/bff1d427ac154d0e9cc98830785f1160 is 175, key is test_row_0/A:col10/1733235328855/Put/seqid=0 2024-12-03T14:15:30,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742240_1416 (size=31255) 2024-12-03T14:15:30,989 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/2fbcf6377a84442abd0666b420c414cc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/2fbcf6377a84442abd0666b420c414cc 2024-12-03T14:15:30,993 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/C of 22c66e6703cd1ab865c50ee7fd3f8725 into 2fbcf6377a84442abd0666b420c414cc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:30,993 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:30,993 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/C, priority=10, startTime=1733235330089; duration=0sec 2024-12-03T14:15:30,993 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:30,994 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:C 2024-12-03T14:15:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:31,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:31,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=531, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/bff1d427ac154d0e9cc98830785f1160 2024-12-03T14:15:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73b3f63de1184f36b1ad0c01a4d19a4b is 50, key is test_row_0/B:col10/1733235328855/Put/seqid=0 2024-12-03T14:15:31,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742241_1417 (size=12301) 2024-12-03T14:15:31,062 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73b3f63de1184f36b1ad0c01a4d19a4b 2024-12-03T14:15:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/ff7342c5d0c6485da0a74575e94df165 is 50, key is test_row_0/C:col10/1733235328855/Put/seqid=0 2024-12-03T14:15:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742242_1418 (size=12301) 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235331067, max timestamp=1733235331089Num uniques 859; 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235331067, max timestamp=1733235331089Num uniques 859; 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235331067, max timestamp=1733235331089Num uniques 859; 2024-12-03T14:15:31,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235331113, max timestamp=1733235331162Num uniques 1168; 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235331113, max timestamp=1733235331162Num uniques 1168; 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733235331113, max timestamp=1733235331162Num uniques 1168; 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T14:15:31,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T14:15:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235391236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235331207, max timestamp=1733235331242Num uniques 1077; 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235331207, max timestamp=1733235331242Num uniques 1077; 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235331207, max timestamp=1733235331242Num uniques 1077; 2024-12-03T14:15:31,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-03T14:15:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235331341, max timestamp=1733235331366Num uniques 1086; 2024-12-03T14:15:31,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:31,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235331341, max timestamp=1733235331366Num uniques 1086; 2024-12-03T14:15:31,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733235331341, max timestamp=1733235331366Num uniques 1086; 2024-12-03T14:15:31,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:31,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T14:15:31,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235391414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733235331403, max timestamp=1733235331420Num uniques 1095; 2024-12-03T14:15:31,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-03T14:15:31,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733235331403, max timestamp=1733235331420Num uniques 1095; 2024-12-03T14:15:31,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733235331403, max timestamp=1733235331420Num uniques 1095; 2024-12-03T14:15:31,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-03T14:15:31,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-03T14:15:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:31,470 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/ff7342c5d0c6485da0a74575e94df165 2024-12-03T14:15:31,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/bff1d427ac154d0e9cc98830785f1160 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160 2024-12-03T14:15:31,476 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160, entries=150, sequenceid=531, filesize=30.5 K 2024-12-03T14:15:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/73b3f63de1184f36b1ad0c01a4d19a4b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b 2024-12-03T14:15:31,480 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b, entries=150, sequenceid=531, filesize=12.0 K 2024-12-03T14:15:31,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/ff7342c5d0c6485da0a74575e94df165 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165 2024-12-03T14:15:31,484 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165, entries=150, sequenceid=531, filesize=12.0 K 2024-12-03T14:15:31,484 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=100.63 KB/103050 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1250ms, sequenceid=531, compaction requested=false 2024-12-03T14:15:31,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:31,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:31,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T14:15:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T14:15:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:31,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=107.34 KB heapSize=301.92 KB 2024-12-03T14:15:31,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:31,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T14:15:31,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-12-03T14:15:31,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120324ac43089a434fd8b763bc183d213d4b_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:31,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.1720 sec 2024-12-03T14:15:31,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742243_1419 (size=14994) 2024-12-03T14:15:31,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235391549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235391558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235391666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235391676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235391876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235391886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:31,897 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:31,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120324ac43089a434fd8b763bc183d213d4b_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120324ac43089a434fd8b763bc183d213d4b_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:31,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/df9f046388bf4ea19d19003b4f9407fb, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:31,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/df9f046388bf4ea19d19003b4f9407fb is 175, key is test_row_0/A:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:31,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742244_1420 (size=39949) 2024-12-03T14:15:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60110 deadline: 1733235391916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:31,923 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235392196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235392196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,305 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=635, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/df9f046388bf4ea19d19003b4f9407fb 2024-12-03T14:15:32,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/71a55311d9084566b53a9bab000ecee4 is 50, key is test_row_0/B:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:32,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742245_1421 (size=12301) 2024-12-03T14:15:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60152 deadline: 1733235392326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,330 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60094 deadline: 1733235392346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,348 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in 22c66e6703cd1ab865c50ee7fd3f8725 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60144 deadline: 1733235392706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60166 deadline: 1733235392706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:32,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/71a55311d9084566b53a9bab000ecee4 2024-12-03T14:15:32,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:32,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:15:32,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/7b74609e96834bcb8ff21d8703682851 is 50, key is test_row_0/C:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:32,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742246_1422 (size=12301) 2024-12-03T14:15:33,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/7b74609e96834bcb8ff21d8703682851 2024-12-03T14:15:33,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/df9f046388bf4ea19d19003b4f9407fb as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb 2024-12-03T14:15:33,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb, entries=200, sequenceid=635, filesize=39.0 K 2024-12-03T14:15:33,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/71a55311d9084566b53a9bab000ecee4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4 2024-12-03T14:15:33,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4, entries=150, sequenceid=635, filesize=12.0 K 2024-12-03T14:15:33,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/7b74609e96834bcb8ff21d8703682851 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851 2024-12-03T14:15:33,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851, entries=150, sequenceid=635, filesize=12.0 K 2024-12-03T14:15:33,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~301.88 KB/309120, currentSize=73.80 KB/75570 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1659ms, sequenceid=635, compaction requested=true 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:33,146 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:33,146 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22c66e6703cd1ab865c50ee7fd3f8725:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:33,146 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:33,146 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:33,146 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/A is initiating minor compaction (all files) 2024-12-03T14:15:33,147 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/B is initiating minor compaction (all files) 2024-12-03T14:15:33,147 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/A in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:33,147 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/B in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:33,147 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=100.6 K 2024-12-03T14:15:33,147 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/1c00d668dad946ef9f0ced567476a7e0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=36.5 K 2024-12-03T14:15:33,147 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:33,147 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb] 2024-12-03T14:15:33,147 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c00d668dad946ef9f0ced567476a7e0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:33,147 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb2e34591d5b4d7890eda54e111bafee, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:33,148 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b3f63de1184f36b1ad0c01a4d19a4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733235328852 2024-12-03T14:15:33,148 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff1d427ac154d0e9cc98830785f1160, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733235328852 2024-12-03T14:15:33,148 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a55311d9084566b53a9bab000ecee4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733235331464 2024-12-03T14:15:33,148 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting df9f046388bf4ea19d19003b4f9407fb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733235331443 2024-12-03T14:15:33,160 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:33,161 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:33,162 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/bfad8d7ecde94d5f82ba42950d7cc268 is 50, key is test_row_0/B:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:33,163 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120326444e8692e249e9801cc470e714238a_22c66e6703cd1ab865c50ee7fd3f8725 store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:33,165 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120326444e8692e249e9801cc470e714238a_22c66e6703cd1ab865c50ee7fd3f8725, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:33,165 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120326444e8692e249e9801cc470e714238a_22c66e6703cd1ab865c50ee7fd3f8725 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:33,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742247_1423 (size=12915) 2024-12-03T14:15:33,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742248_1424 (size=4469) 2024-12-03T14:15:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T14:15:33,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:33,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-03T14:15:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:33,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:33,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:33,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:33,517 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-12-03T14:15:33,517 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,517 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,517 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,518 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-12-03T14:15:33,518 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,518 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,518 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,521 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-12-03T14:15:33,521 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,521 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,521 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-12-03T14:15:33,521 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,521 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,522 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,522 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,523 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-12-03T14:15:33,523 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,523 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,523 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,576 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:33,577 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/564b580994c8467fb04666667fc4936b is 175, key is test_row_0/A:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:33,578 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/bfad8d7ecde94d5f82ba42950d7cc268 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/bfad8d7ecde94d5f82ba42950d7cc268 2024-12-03T14:15:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742249_1425 (size=31869) 2024-12-03T14:15:33,586 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/B of 22c66e6703cd1ab865c50ee7fd3f8725 into bfad8d7ecde94d5f82ba42950d7cc268(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:33,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:33,586 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/B, priority=13, startTime=1733235333146; duration=0sec 2024-12-03T14:15:33,587 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:33,587 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:B 2024-12-03T14:15:33,587 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:33,587 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:33,587 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): 22c66e6703cd1ab865c50ee7fd3f8725/C is initiating minor compaction (all files) 2024-12-03T14:15:33,588 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22c66e6703cd1ab865c50ee7fd3f8725/C in TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:33,588 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/2fbcf6377a84442abd0666b420c414cc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp, totalSize=36.5 K 2024-12-03T14:15:33,588 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fbcf6377a84442abd0666b420c414cc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1733235327604 2024-12-03T14:15:33,588 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7342c5d0c6485da0a74575e94df165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733235328852 2024-12-03T14:15:33,588 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b74609e96834bcb8ff21d8703682851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1733235331464 2024-12-03T14:15:33,593 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22c66e6703cd1ab865c50ee7fd3f8725#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:33,593 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/d0d286b8eaf54d808b6d5e5a165bc1f2 is 50, key is test_row_0/C:col10/1733235331464/Put/seqid=0 2024-12-03T14:15:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742250_1426 (size=12915) 2024-12-03T14:15:33,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:33,641 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:33,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:33,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba07caa8dcc44f44ae5a905701417d0a_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235331548/Put/seqid=0 2024-12-03T14:15:33,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742251_1427 (size=12454) 2024-12-03T14:15:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:33,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. as already flushing 2024-12-03T14:15:33,728 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-12-03T14:15:33,728 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-12-03T14:15:33,728 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,728 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:33,728 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,728 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,728 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,728 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:33,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:33,990 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/564b580994c8467fb04666667fc4936b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/564b580994c8467fb04666667fc4936b 2024-12-03T14:15:33,995 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/A of 22c66e6703cd1ab865c50ee7fd3f8725 into 564b580994c8467fb04666667fc4936b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:33,995 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:33,995 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/A, priority=13, startTime=1733235333146; duration=0sec 2024-12-03T14:15:33,995 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:33,995 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:A 2024-12-03T14:15:34,001 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/d0d286b8eaf54d808b6d5e5a165bc1f2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d0d286b8eaf54d808b6d5e5a165bc1f2 2024-12-03T14:15:34,007 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22c66e6703cd1ab865c50ee7fd3f8725/C of 22c66e6703cd1ab865c50ee7fd3f8725 into d0d286b8eaf54d808b6d5e5a165bc1f2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:34,007 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:34,007 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., storeName=22c66e6703cd1ab865c50ee7fd3f8725/C, priority=13, startTime=1733235333146; duration=0sec 2024-12-03T14:15:34,007 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:34,007 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22c66e6703cd1ab865c50ee7fd3f8725:C 2024-12-03T14:15:34,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:34,053 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ba07caa8dcc44f44ae5a905701417d0a_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba07caa8dcc44f44ae5a905701417d0a_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e4f791837cd445f9a29a98df5a0a802b, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e4f791837cd445f9a29a98df5a0a802b is 175, key is test_row_0/A:col10/1733235331548/Put/seqid=0 2024-12-03T14:15:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742252_1428 (size=31255) 2024-12-03T14:15:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:34,458 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=650, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e4f791837cd445f9a29a98df5a0a802b 2024-12-03T14:15:34,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/57b87bd1a39845debc16ecff52f71665 is 50, key is test_row_0/B:col10/1733235331548/Put/seqid=0 2024-12-03T14:15:34,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742253_1429 (size=12301) 2024-12-03T14:15:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:34,873 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/57b87bd1a39845debc16ecff52f71665 2024-12-03T14:15:34,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/4aa8a01785364631ac12dd132d898812 is 50, key is test_row_0/C:col10/1733235331548/Put/seqid=0 2024-12-03T14:15:34,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742254_1430 (size=12301) 2024-12-03T14:15:35,287 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/4aa8a01785364631ac12dd132d898812 2024-12-03T14:15:35,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/e4f791837cd445f9a29a98df5a0a802b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e4f791837cd445f9a29a98df5a0a802b 2024-12-03T14:15:35,296 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e4f791837cd445f9a29a98df5a0a802b, entries=150, sequenceid=650, filesize=30.5 K 2024-12-03T14:15:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/57b87bd1a39845debc16ecff52f71665 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/57b87bd1a39845debc16ecff52f71665 2024-12-03T14:15:35,301 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/57b87bd1a39845debc16ecff52f71665, entries=150, sequenceid=650, filesize=12.0 K 2024-12-03T14:15:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/4aa8a01785364631ac12dd132d898812 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/4aa8a01785364631ac12dd132d898812 2024-12-03T14:15:35,306 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/4aa8a01785364631ac12dd132d898812, entries=150, sequenceid=650, filesize=12.0 K 2024-12-03T14:15:35,307 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1666ms, sequenceid=650, compaction requested=false 2024-12-03T14:15:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 22c66e6703cd1ab865c50ee7fd3f8725: 2024-12-03T14:15:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-03T14:15:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-03T14:15:35,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T14:15:35,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8190 sec 2024-12-03T14:15:35,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8250 sec 2024-12-03T14:15:35,458 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:15:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T14:15:35,638 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:35,969 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-12-03T14:15:35,969 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:35,969 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:35,969 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:35,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:40,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb] to archive 2024-12-03T14:15:40,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:40,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/cd3122c61f254ce1baabade30816e7de 2024-12-03T14:15:40,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/561860645f524b87b598a8032e5038cb 2024-12-03T14:15:40,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/ffac9e483af54f66b8d7c26934f8d61d 2024-12-03T14:15:40,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/b0c24443c08e4fc68e5933004fb8a8fd 2024-12-03T14:15:40,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/758aed63ebe84e7a81ceae475bda8323 2024-12-03T14:15:40,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/20ae75530dc541c3a9478727addc178f 2024-12-03T14:15:40,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/2a105666e45040a78784ba658073baa3 2024-12-03T14:15:40,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fe1bb87bb5b045fc992313b4d6431e86 2024-12-03T14:15:40,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/c2e0210a4a27493eb2c2ead0f4af6638 2024-12-03T14:15:40,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/d286f9643f864901bbe9535ad2730837 2024-12-03T14:15:40,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/603d097fb4ab40e1befec35cc1d5977c 2024-12-03T14:15:40,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/9daae805286b4b199845e8b53ebced70 2024-12-03T14:15:40,032 WARN [regionserver/3920e6a7c3b5:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-12-03T14:15:40,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/eddf5bea82b84d9a8e720ce82888227a 2024-12-03T14:15:40,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e7ec547df4254212bcb3339787fe76fb 2024-12-03T14:15:40,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/8ca7250e5b3b4d5ba3367f1a92ea9e26 2024-12-03T14:15:40,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/fb2e34591d5b4d7890eda54e111bafee 2024-12-03T14:15:40,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/bff1d427ac154d0e9cc98830785f1160 2024-12-03T14:15:40,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/df9f046388bf4ea19d19003b4f9407fb 2024-12-03T14:15:40,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d30a7a071ac74c4e8334e219b4e28caf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/2a8ce41b520143e2af72742426960919, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73cf838aa59841fda29039de20ae2caa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/1c00d668dad946ef9f0ced567476a7e0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4] to archive 2024-12-03T14:15:40,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:40,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/a8fafe4c8d074bb380d1f1037bdcc0b4 2024-12-03T14:15:40,045 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/964885897d33463d919244502a1f3b92 2024-12-03T14:15:40,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d30a7a071ac74c4e8334e219b4e28caf to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d30a7a071ac74c4e8334e219b4e28caf 2024-12-03T14:15:40,047 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/743b32bef6d6439b91e4e689efe9f1dd 2024-12-03T14:15:40,048 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/62323db681f44075b1d08072ab963ce6 2024-12-03T14:15:40,049 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/2a8ce41b520143e2af72742426960919 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/2a8ce41b520143e2af72742426960919 2024-12-03T14:15:40,050 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/4a947b2100994cfa9b31964fca148e63 2024-12-03T14:15:40,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/90bf5bfc4db04667aa9452955e24e2d1 2024-12-03T14:15:40,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73cf838aa59841fda29039de20ae2caa to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73cf838aa59841fda29039de20ae2caa 2024-12-03T14:15:40,052 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/d38a3e94ae1042f5b8c0f657c58c0c55 2024-12-03T14:15:40,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0475e3c39bf34379a4e657608b0cb32d 2024-12-03T14:15:40,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/9e48d3b352684d20b4f8d9a8000b737a 2024-12-03T14:15:40,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/e3108a0c08e14698886babb28bb7d80c 2024-12-03T14:15:40,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73fe96cd4e224924a278787fcbaa7c65 2024-12-03T14:15:40,057 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/1c00d668dad946ef9f0ced567476a7e0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/1c00d668dad946ef9f0ced567476a7e0 2024-12-03T14:15:40,058 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/0164b345ea50475dbe3ecf016a6b34b2 2024-12-03T14:15:40,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/73b3f63de1184f36b1ad0c01a4d19a4b 2024-12-03T14:15:40,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/71a55311d9084566b53a9bab000ecee4 2024-12-03T14:15:40,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/00bc4c968ae04f85a085f9589f415efd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/12817a3fb48b4976a6a3bf3930632981, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/9c4d9d131bc540c09642c91f3d8d36e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/2fbcf6377a84442abd0666b420c414cc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851] to archive 2024-12-03T14:15:40,064 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:15:40,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/c04c7b0d5474411fa9eeeb3f76a91eca 2024-12-03T14:15:40,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/fbf898e0fe584c139335fb7ad35ff2e3 2024-12-03T14:15:40,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/00bc4c968ae04f85a085f9589f415efd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/00bc4c968ae04f85a085f9589f415efd 2024-12-03T14:15:40,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1b9c0f07d2dc462cb6ecc8c400bb2439 2024-12-03T14:15:40,069 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfbb87c14b4145d5b6445115e29605ec 2024-12-03T14:15:40,070 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/12817a3fb48b4976a6a3bf3930632981 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/12817a3fb48b4976a6a3bf3930632981 2024-12-03T14:15:40,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a7f95b6f6f4d4d099bf35f5e3f59952c 2024-12-03T14:15:40,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a9aa62f00a244f3a9eef0aad047c5ed9 2024-12-03T14:15:40,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/9c4d9d131bc540c09642c91f3d8d36e1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/9c4d9d131bc540c09642c91f3d8d36e1 2024-12-03T14:15:40,073 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/a05e587f3f224fb6ac933b2f31b84241 2024-12-03T14:15:40,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/cd8c7075187b48b99dcaccdbe262b3ac 2024-12-03T14:15:40,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/50ff93e400cd433199e16c250ceb6522 2024-12-03T14:15:40,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/1629d508b8674249b6937c95f515cfe8 2024-12-03T14:15:40,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d076b2b037504a91a07392420f2e7521 2024-12-03T14:15:40,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/2fbcf6377a84442abd0666b420c414cc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/2fbcf6377a84442abd0666b420c414cc 2024-12-03T14:15:40,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/bfc6014315d748e9b6c5dcfc4c409150 2024-12-03T14:15:40,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/ff7342c5d0c6485da0a74575e94df165 2024-12-03T14:15:40,079 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3920e6a7c3b5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/7b74609e96834bcb8ff21d8703682851 2024-12-03T14:15:42,398 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-12-03T14:15:42,398 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:42,398 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,399 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:42,408 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-12-03T14:15:42,408 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:15:42,408 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,409 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:42,410 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1370 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4108 rows 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1375 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4119 rows 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1364 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4088 rows 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1367 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4100 rows 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1365 2024-12-03T14:15:42,411 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4090 rows 2024-12-03T14:15:42,411 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:15:42,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:15:42,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:42,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:15:42,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:15:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:42,418 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235342418"}]},"ts":"1733235342418"} 2024-12-03T14:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:42,420 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:15:42,420 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:15:42,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:15:42,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, UNASSIGN}] 2024-12-03T14:15:42,425 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, UNASSIGN 2024-12-03T14:15:42,427 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:42,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, UNASSIGN because future has completed 2024-12-03T14:15:42,429 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:15:42,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:42,581 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:42,581 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:15:42,581 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 22c66e6703cd1ab865c50ee7fd3f8725, disabling compactions & flushes 2024-12-03T14:15:42,582 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. after waiting 0 ms 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:42,582 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 22c66e6703cd1ab865c50ee7fd3f8725 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=A 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=B 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 22c66e6703cd1ab865c50ee7fd3f8725, store=C 2024-12-03T14:15:42,582 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:42,588 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c7227e693c8a4976b46bd712bcefa00d_22c66e6703cd1ab865c50ee7fd3f8725 is 50, key is test_row_0/A:col10/1733235333726/Put/seqid=0 2024-12-03T14:15:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742255_1431 (size=12454) 2024-12-03T14:15:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:42,992 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:42,996 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c7227e693c8a4976b46bd712bcefa00d_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c7227e693c8a4976b46bd712bcefa00d_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:42,997 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/49b5b52b91c2477ba7ac65ca3a648a93, store: [table=TestAcidGuarantees family=A region=22c66e6703cd1ab865c50ee7fd3f8725] 2024-12-03T14:15:42,997 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/49b5b52b91c2477ba7ac65ca3a648a93 is 175, key is test_row_0/A:col10/1733235333726/Put/seqid=0 2024-12-03T14:15:43,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742256_1432 (size=31255) 2024-12-03T14:15:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:43,402 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=660, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/49b5b52b91c2477ba7ac65ca3a648a93 2024-12-03T14:15:43,407 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/15be4a56ef344b89b0e45bbbbfc8e0fd is 50, key is test_row_0/B:col10/1733235333726/Put/seqid=0 2024-12-03T14:15:43,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742257_1433 (size=12301) 2024-12-03T14:15:43,416 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/15be4a56ef344b89b0e45bbbbfc8e0fd 2024-12-03T14:15:43,422 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/e680f373511140f5a206f2bff5459c7b is 50, key is test_row_0/C:col10/1733235333726/Put/seqid=0 2024-12-03T14:15:43,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742258_1434 (size=12301) 2024-12-03T14:15:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:43,827 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=660 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/e680f373511140f5a206f2bff5459c7b 2024-12-03T14:15:43,837 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/A/49b5b52b91c2477ba7ac65ca3a648a93 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/49b5b52b91c2477ba7ac65ca3a648a93 2024-12-03T14:15:43,841 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/49b5b52b91c2477ba7ac65ca3a648a93, entries=150, sequenceid=660, filesize=30.5 K 2024-12-03T14:15:43,842 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/B/15be4a56ef344b89b0e45bbbbfc8e0fd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/15be4a56ef344b89b0e45bbbbfc8e0fd 2024-12-03T14:15:43,844 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/15be4a56ef344b89b0e45bbbbfc8e0fd, entries=150, sequenceid=660, filesize=12.0 K 2024-12-03T14:15:43,845 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/.tmp/C/e680f373511140f5a206f2bff5459c7b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/e680f373511140f5a206f2bff5459c7b 2024-12-03T14:15:43,847 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/e680f373511140f5a206f2bff5459c7b, entries=150, sequenceid=660, filesize=12.0 K 2024-12-03T14:15:43,848 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1266ms, sequenceid=660, compaction requested=true 2024-12-03T14:15:43,851 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits/663.seqid, newMaxSeqId=663, maxSeqId=4 2024-12-03T14:15:43,852 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. 2024-12-03T14:15:43,852 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 22c66e6703cd1ab865c50ee7fd3f8725: Waiting for close lock at 1733235342581Running coprocessor pre-close hooks at 1733235342581Disabling compacts and flushes for region at 1733235342581Disabling writes for close at 1733235342582 (+1 ms)Obtaining lock to block concurrent updates at 1733235342582Preparing flush snapshotting stores in 22c66e6703cd1ab865c50ee7fd3f8725 at 1733235342582Finished memstore snapshotting TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733235342582Flushing stores of TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725. at 1733235342583 (+1 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/A: creating writer at 1733235342583Flushing mob file 22c66e6703cd1ab865c50ee7fd3f8725/A: appending metadata at 1733235342588 (+5 ms)Flushing mob file 22c66e6703cd1ab865c50ee7fd3f8725/A: closing flushed file at 1733235342588Flushing 22c66e6703cd1ab865c50ee7fd3f8725/A: appending metadata at 1733235342997 (+409 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/A: closing flushed file at 1733235342997Flushing 22c66e6703cd1ab865c50ee7fd3f8725/B: creating writer at 1733235343405 (+408 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/B: appending metadata at 1733235343407 (+2 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/B: closing flushed file at 1733235343407Flushing 22c66e6703cd1ab865c50ee7fd3f8725/C: creating writer at 1733235343419 (+12 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/C: appending metadata at 1733235343422 (+3 ms)Flushing 22c66e6703cd1ab865c50ee7fd3f8725/C: closing flushed file at 1733235343422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57aef79d: reopening flushed file at 1733235343835 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b350be: reopening flushed file at 1733235343841 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@716fd738: reopening flushed file at 1733235343844 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 22c66e6703cd1ab865c50ee7fd3f8725 in 1266ms, sequenceid=660, compaction requested=true at 1733235343848 (+4 ms)Writing region close event to WAL at 1733235343849 (+1 ms)Running coprocessor post-close hooks at 1733235343852 (+3 ms)Closed at 1733235343852 2024-12-03T14:15:43,854 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:43,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=22c66e6703cd1ab865c50ee7fd3f8725, regionState=CLOSED 2024-12-03T14:15:43,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:43,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T14:15:43,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 22c66e6703cd1ab865c50ee7fd3f8725, server=3920e6a7c3b5,42177,1733235217010 in 1.4260 sec 2024-12-03T14:15:43,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-03T14:15:43,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22c66e6703cd1ab865c50ee7fd3f8725, UNASSIGN in 1.4330 sec 2024-12-03T14:15:43,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T14:15:43,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4370 sec 2024-12-03T14:15:43,861 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235343860"}]},"ts":"1733235343860"} 2024-12-03T14:15:43,861 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:15:43,862 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:15:43,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4490 sec 2024-12-03T14:15:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T14:15:44,576 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:44,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:15:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T14:15:44,580 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,582 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits] 2024-12-03T14:15:44,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/49b5b52b91c2477ba7ac65ca3a648a93 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/49b5b52b91c2477ba7ac65ca3a648a93 2024-12-03T14:15:44,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/564b580994c8467fb04666667fc4936b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/564b580994c8467fb04666667fc4936b 2024-12-03T14:15:44,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e4f791837cd445f9a29a98df5a0a802b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/A/e4f791837cd445f9a29a98df5a0a802b 2024-12-03T14:15:44,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/15be4a56ef344b89b0e45bbbbfc8e0fd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/15be4a56ef344b89b0e45bbbbfc8e0fd 2024-12-03T14:15:44,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/57b87bd1a39845debc16ecff52f71665 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/57b87bd1a39845debc16ecff52f71665 2024-12-03T14:15:44,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/bfad8d7ecde94d5f82ba42950d7cc268 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/B/bfad8d7ecde94d5f82ba42950d7cc268 2024-12-03T14:15:44,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/4aa8a01785364631ac12dd132d898812 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/4aa8a01785364631ac12dd132d898812 2024-12-03T14:15:44,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d0d286b8eaf54d808b6d5e5a165bc1f2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/d0d286b8eaf54d808b6d5e5a165bc1f2 2024-12-03T14:15:44,602 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/e680f373511140f5a206f2bff5459c7b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/C/e680f373511140f5a206f2bff5459c7b 2024-12-03T14:15:44,605 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits/663.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725/recovered.edits/663.seqid 2024-12-03T14:15:44,605 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,605 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:15:44,606 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:15:44,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T14:15:44,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f5296c9bdb649a29f4abb010b0e20b7_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031f5296c9bdb649a29f4abb010b0e20b7_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,611 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120324ac43089a434fd8b763bc183d213d4b_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120324ac43089a434fd8b763bc183d213d4b_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330cdae902bfa4c059d95d9b806c50bcf_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330cdae902bfa4c059d95d9b806c50bcf_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,613 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360af6054ee8a45369e9d0a765fcea93d_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120360af6054ee8a45369e9d0a765fcea93d_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,615 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203629f6e8054c245d5a886389d4efbcfb6_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203629f6e8054c245d5a886389d4efbcfb6_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,616 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120366b49284d40749d0a39d237b6be28434_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120366b49284d40749d0a39d237b6be28434_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,617 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037a097c1cea21443fb47689f6b0fc3c23_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037a097c1cea21443fb47689f6b0fc3c23_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038be5b91551e6456dab81ae0db3b101aa_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038be5b91551e6456dab81ae0db3b101aa_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa02dd3fbeee445f9d30fcaedd3490f3_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aa02dd3fbeee445f9d30fcaedd3490f3_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,621 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ac6a966738d44655a7106c06ce7906ae_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ac6a966738d44655a7106c06ce7906ae_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,623 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b068ea7aed8147ffa7312d4b005a4b3e_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b068ea7aed8147ffa7312d4b005a4b3e_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b719ff75724944359936148a1cd1e316_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b719ff75724944359936148a1cd1e316_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba07caa8dcc44f44ae5a905701417d0a_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ba07caa8dcc44f44ae5a905701417d0a_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c7227e693c8a4976b46bd712bcefa00d_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c7227e693c8a4976b46bd712bcefa00d_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,628 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e198ffb09f654e73898ee3d7f196c8f2_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e198ffb09f654e73898ee3d7f196c8f2_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f29ff617b3ae4433aac04b7e43dea374_22c66e6703cd1ab865c50ee7fd3f8725 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f29ff617b3ae4433aac04b7e43dea374_22c66e6703cd1ab865c50ee7fd3f8725 2024-12-03T14:15:44,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:15:44,631 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,633 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:15:44,634 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:15:44,635 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,635 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:15:44,635 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235344635"}]},"ts":"9223372036854775807"} 2024-12-03T14:15:44,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:15:44,637 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 22c66e6703cd1ab865c50ee7fd3f8725, NAME => 'TestAcidGuarantees,,1733235310397.22c66e6703cd1ab865c50ee7fd3f8725.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:15:44,637 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:15:44,637 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235344637"}]},"ts":"9223372036854775807"} 2024-12-03T14:15:44,638 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:15:44,639 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-12-03T14:15:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T14:15:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:15:44,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:44,700 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 249), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=10069 (was 11020) 2024-12-03T14:15:44,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=10068 2024-12-03T14:15:44,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:15:44,710 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:15:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:15:44,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:15:44,712 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:44,713 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-12-03T14:15:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:44,713 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:15:44,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742259_1435 (size=1061) 2024-12-03T14:15:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:45,124 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:15:45,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742260_1436 (size=53) 2024-12-03T14:15:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:45,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:15:45,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e0c0e6a2f069c0cb795f0cc0acfdf529, disabling compactions & flushes 2024-12-03T14:15:45,533 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. after waiting 0 ms 2024-12-03T14:15:45,533 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,533 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,534 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e0c0e6a2f069c0cb795f0cc0acfdf529: Waiting for close lock at 1733235345533Disabling compacts and flushes for region at 1733235345533Disabling writes for close at 1733235345533Writing region close event to WAL at 1733235345533Closed at 1733235345533 2024-12-03T14:15:45,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:15:45,535 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235345535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235345535"}]},"ts":"1733235345535"} 2024-12-03T14:15:45,536 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:15:45,537 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:15:45,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235345537"}]},"ts":"1733235345537"} 2024-12-03T14:15:45,539 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:15:45,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, ASSIGN}] 2024-12-03T14:15:45,540 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, ASSIGN 2024-12-03T14:15:45,541 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:15:45,692 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e0c0e6a2f069c0cb795f0cc0acfdf529, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:45,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, ASSIGN because future has completed 2024-12-03T14:15:45,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:15:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:45,851 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,852 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:15:45,853 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,853 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:15:45,853 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,853 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,856 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,857 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:45,858 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c0e6a2f069c0cb795f0cc0acfdf529 columnFamilyName A 2024-12-03T14:15:45,858 DEBUG [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:45,858 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(327): Store=e0c0e6a2f069c0cb795f0cc0acfdf529/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:45,859 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,860 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:45,860 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c0e6a2f069c0cb795f0cc0acfdf529 columnFamilyName B 2024-12-03T14:15:45,860 DEBUG [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:45,861 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(327): Store=e0c0e6a2f069c0cb795f0cc0acfdf529/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:45,861 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,862 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:15:45,862 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c0e6a2f069c0cb795f0cc0acfdf529 columnFamilyName C 2024-12-03T14:15:45,862 DEBUG [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:15:45,862 INFO [StoreOpener-e0c0e6a2f069c0cb795f0cc0acfdf529-1 {}] regionserver.HStore(327): Store=e0c0e6a2f069c0cb795f0cc0acfdf529/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:15:45,863 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,863 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,863 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,864 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,865 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,865 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,865 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:15:45,866 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,868 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:15:45,868 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened e0c0e6a2f069c0cb795f0cc0acfdf529; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69503126, jitterRate=0.03567728400230408}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:15:45,868 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:45,869 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for e0c0e6a2f069c0cb795f0cc0acfdf529: Running coprocessor pre-open hook at 1733235345853Writing region info on filesystem at 1733235345853Initializing all the Stores at 1733235345855 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235345855Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235345855Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235345855Cleaning up temporary data from old regions at 1733235345865 (+10 ms)Running coprocessor post-open hooks at 1733235345868 (+3 ms)Region opened successfully at 1733235345869 (+1 ms) 2024-12-03T14:15:45,869 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., pid=114, masterSystemTime=1733235345846 2024-12-03T14:15:45,871 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,871 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:45,871 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=e0c0e6a2f069c0cb795f0cc0acfdf529, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:45,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:15:45,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T14:15:45,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 in 179 msec 2024-12-03T14:15:45,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-03T14:15:45,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, ASSIGN in 336 msec 2024-12-03T14:15:45,878 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:15:45,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235345878"}]},"ts":"1733235345878"} 2024-12-03T14:15:45,879 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:15:45,880 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:15:45,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-12-03T14:15:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T14:15:46,856 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:46,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,858 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4db38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497a54ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,865 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c583ddf 2024-12-03T14:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:15:46,867 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:15:46,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a148e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e936584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,874 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2db440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9249511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1708d711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2bdd09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,880 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d56d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@889d280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0c61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,884 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260b2af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,885 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d613ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a66dca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,888 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c67a528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e277b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,890 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb4ca4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6122bde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,892 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36235261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,894 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb62ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:15:46,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:15:46,895 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4423fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:15:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:15:46,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:15:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217c446d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:15:46,897 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,897 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,897 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,897 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,897 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,898 DEBUG [Thread-1960 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,899 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,900 DEBUG [Thread-1973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,901 DEBUG [Thread-1971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,905 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:15:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-03T14:15:46,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,907 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T14:15:46,907 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:46,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:46,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:15:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:15:46,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:15:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2] 2024-12-03T14:15:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:46,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:46,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c23b9083da20456c9d06c1a3288f0766 is 50, key is test_row_0/A:col10/1733235346911/Put/seqid=0 2024-12-03T14:15:46,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235346928, max timestamp=1733235346936Num uniques 859; 2024-12-03T14:15:46,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:46,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235346928, max timestamp=1733235346936Num uniques 859; 2024-12-03T14:15:46,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:46,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235346928, max timestamp=1733235346936Num uniques 859; 2024-12-03T14:15:46,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:46,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742261_1437 (size=14341) 2024-12-03T14:15:46,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c23b9083da20456c9d06c1a3288f0766 2024-12-03T14:15:46,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6c6a1b824bd42528ae91e47e38cb8de is 50, key is test_row_0/B:col10/1733235346911/Put/seqid=0 2024-12-03T14:15:46,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742262_1438 (size=12001) 2024-12-03T14:15:46,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6c6a1b824bd42528ae91e47e38cb8de 2024-12-03T14:15:46,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:46,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235406968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235406967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235406969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235406969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235406969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:46,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/355116a291b94c1e9e9420680fe698bd is 50, key is test_row_0/C:col10/1733235346911/Put/seqid=0 2024-12-03T14:15:46,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742263_1439 (size=12001) 2024-12-03T14:15:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T14:15:47,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T14:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235407085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235407086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235407086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235407086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235407086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T14:15:47,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:47,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T14:15:47,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235407296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235407296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235407296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235407296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235407296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T14:15:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:47,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/355116a291b94c1e9e9420680fe698bd 2024-12-03T14:15:47,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c23b9083da20456c9d06c1a3288f0766 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766 2024-12-03T14:15:47,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766, entries=200, sequenceid=11, filesize=14.0 K 2024-12-03T14:15:47,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6c6a1b824bd42528ae91e47e38cb8de as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de 2024-12-03T14:15:47,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de, entries=150, sequenceid=11, filesize=11.7 K 2024-12-03T14:15:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/355116a291b94c1e9e9420680fe698bd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd 2024-12-03T14:15:47,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd, entries=150, sequenceid=11, filesize=11.7 K 2024-12-03T14:15:47,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 487ms, sequenceid=11, compaction requested=false 2024-12-03T14:15:47,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:47,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:47,520 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:47,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/94cfd7885b434dbdbb6c8924516d4bed is 50, key is test_row_0/A:col10/1733235346968/Put/seqid=0 2024-12-03T14:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T14:15:47,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742264_1440 (size=12001) 2024-12-03T14:15:47,544 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/94cfd7885b434dbdbb6c8924516d4bed 2024-12-03T14:15:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/a1ed96797b784b3d820d5891a23c9686 is 50, key is test_row_0/B:col10/1733235346968/Put/seqid=0 2024-12-03T14:15:47,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742265_1441 (size=12001) 2024-12-03T14:15:47,592 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/a1ed96797b784b3d820d5891a23c9686 2024-12-03T14:15:47,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/2bae927f39ba4858b7e85182230cad4b is 50, key is test_row_0/C:col10/1733235346968/Put/seqid=0 2024-12-03T14:15:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742266_1442 (size=12001) 2024-12-03T14:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:47,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235407611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235407612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235407612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235407613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235407614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235407726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235407726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235407726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235407727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235407727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235407936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235407936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235407937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235407937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235407937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,006 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/2bae927f39ba4858b7e85182230cad4b 2024-12-03T14:15:48,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/94cfd7885b434dbdbb6c8924516d4bed as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed 2024-12-03T14:15:48,012 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T14:15:48,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/a1ed96797b784b3d820d5891a23c9686 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686 2024-12-03T14:15:48,015 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T14:15:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/2bae927f39ba4858b7e85182230cad4b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b 2024-12-03T14:15:48,020 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T14:15:48,021 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 501ms, sequenceid=47, compaction requested=false 2024-12-03T14:15:48,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:48,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:48,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-03T14:15:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-03T14:15:48,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T14:15:48,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-12-03T14:15:48,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.1240 sec 2024-12-03T14:15:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T14:15:48,046 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:48,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T14:15:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T14:15:48,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:48,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:48,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T14:15:48,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:48,201 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/811fa6344977452a92dfd98b105e2a5a is 50, key is test_row_0/A:col10/1733235347613/Put/seqid=0 2024-12-03T14:15:48,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742267_1443 (size=9657) 2024-12-03T14:15:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235348257, max timestamp=1733235348267Num uniques 859; 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235348257, max timestamp=1733235348267Num uniques 859; 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733235348257, max timestamp=1733235348267Num uniques 859; 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:48,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T14:15:48,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235408288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235408288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235408297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235408297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T14:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235408406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235408406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235408406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235408416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235408416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,622 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/811fa6344977452a92dfd98b105e2a5a 2024-12-03T14:15:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235408616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235408617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235408617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235408627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235408627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/371cbd2836734b9d83d7fc93174cb3f1 is 50, key is test_row_0/B:col10/1733235347613/Put/seqid=0 2024-12-03T14:15:48,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742268_1444 (size=9657) 2024-12-03T14:15:48,651 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/371cbd2836734b9d83d7fc93174cb3f1 2024-12-03T14:15:48,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/b52900265a7e4cb3a2f4913ee2427f4c is 50, key is test_row_0/C:col10/1733235347613/Put/seqid=0 2024-12-03T14:15:48,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742269_1445 (size=9657) 2024-12-03T14:15:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T14:15:48,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235408928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235408936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235408936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235408946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,062 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/b52900265a7e4cb3a2f4913ee2427f4c 2024-12-03T14:15:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/811fa6344977452a92dfd98b105e2a5a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a 2024-12-03T14:15:49,086 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a, entries=100, sequenceid=56, filesize=9.4 K 2024-12-03T14:15:49,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/371cbd2836734b9d83d7fc93174cb3f1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1 2024-12-03T14:15:49,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1, entries=100, sequenceid=56, filesize=9.4 K 2024-12-03T14:15:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/b52900265a7e4cb3a2f4913ee2427f4c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c 2024-12-03T14:15:49,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c, entries=100, sequenceid=56, filesize=9.4 K 2024-12-03T14:15:49,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 898ms, sequenceid=56, compaction requested=true 2024-12-03T14:15:49,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:49,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:49,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T14:15:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T14:15:49,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T14:15:49,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0540 sec 2024-12-03T14:15:49,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0590 sec 2024-12-03T14:15:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T14:15:49,186 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:49,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T14:15:49,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T14:15:49,189 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:49,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T14:15:49,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:49,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:49,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/b131f8aa067a4fd7bb98e2cf7b9b34dc is 50, key is test_row_0/A:col10/1733235348286/Put/seqid=0 2024-12-03T14:15:49,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742270_1446 (size=12001) 2024-12-03T14:15:49,366 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:15:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:49,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235409464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235409466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235409468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235409468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235409473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T14:15:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235409576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235409586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235409586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,752 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/b131f8aa067a4fd7bb98e2cf7b9b34dc 2024-12-03T14:15:49,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/753e84cba117493ba0050ccddfa78bc7 is 50, key is test_row_0/B:col10/1733235348286/Put/seqid=0 2024-12-03T14:15:49,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235409786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742271_1447 (size=12001) 2024-12-03T14:15:49,809 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/753e84cba117493ba0050ccddfa78bc7 2024-12-03T14:15:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235409807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235409807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:49,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T14:15:49,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/1f51aa93f43b4f68b4bccf0d0376f37a is 50, key is test_row_0/C:col10/1733235348286/Put/seqid=0 2024-12-03T14:15:49,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742272_1448 (size=12001) 2024-12-03T14:15:49,849 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/1f51aa93f43b4f68b4bccf0d0376f37a 2024-12-03T14:15:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/b131f8aa067a4fd7bb98e2cf7b9b34dc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc 2024-12-03T14:15:49,864 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc, entries=150, sequenceid=92, filesize=11.7 K 2024-12-03T14:15:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/753e84cba117493ba0050ccddfa78bc7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7 2024-12-03T14:15:49,870 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7, entries=150, sequenceid=92, filesize=11.7 K 2024-12-03T14:15:49,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/1f51aa93f43b4f68b4bccf0d0376f37a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a 2024-12-03T14:15:49,879 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a, entries=150, sequenceid=92, filesize=11.7 K 2024-12-03T14:15:49,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 538ms, sequenceid=92, compaction requested=true 2024-12-03T14:15:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:49,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T14:15:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T14:15:49,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T14:15:49,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-12-03T14:15:49,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 700 msec 2024-12-03T14:15:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:50,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:50,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/bd742f2ad0524acf96980f88a7922459 is 50, key is test_row_0/A:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:50,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742273_1449 (size=14341) 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235350154, max timestamp=1733235350168Num uniques 859; 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235350154, max timestamp=1733235350168Num uniques 859; 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235350154, max timestamp=1733235350168Num uniques 859; 2024-12-03T14:15:50,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:50,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235410222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235410222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235410225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T14:15:50,326 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:50,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T14:15:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:50,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:50,329 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:50,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:50,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235410337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235410337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235410346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:50,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T14:15:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235410496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235410496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,504 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,504 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/bd742f2ad0524acf96980f88a7922459 2024-12-03T14:15:50,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ab37980f4d98426ea92eee94363dee3e is 50, key is test_row_0/B:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:50,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742274_1450 (size=12001) 2024-12-03T14:15:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235410546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235410546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235410556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T14:15:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:50,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T14:15:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235410866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235410866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235410876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:50,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ab37980f4d98426ea92eee94363dee3e 2024-12-03T14:15:50,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/e61d2e00f9784e84bd2c2536ac6da291 is 50, key is test_row_0/C:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:50,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T14:15:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:50,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742275_1451 (size=12001) 2024-12-03T14:15:50,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/e61d2e00f9784e84bd2c2536ac6da291 2024-12-03T14:15:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/bd742f2ad0524acf96980f88a7922459 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459 2024-12-03T14:15:50,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459, entries=200, sequenceid=102, filesize=14.0 K 2024-12-03T14:15:50,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ab37980f4d98426ea92eee94363dee3e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e 2024-12-03T14:15:50,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e, entries=150, sequenceid=102, filesize=11.7 K 2024-12-03T14:15:50,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/e61d2e00f9784e84bd2c2536ac6da291 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291 2024-12-03T14:15:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:50,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291, entries=150, sequenceid=102, filesize=11.7 K 2024-12-03T14:15:50,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 843ms, sequenceid=102, compaction requested=true 2024-12-03T14:15:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:50,959 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:50,959 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:15:50,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:50,961 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62341 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:15:50,961 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:15:50,961 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:15:50,961 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:15:50,961 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,961 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,961 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=60.9 K 2024-12-03T14:15:50,961 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=56.3 K 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23b9083da20456c9d06c1a3288f0766, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733235346906 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c6a1b824bd42528ae91e47e38cb8de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733235346906 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ed96797b784b3d820d5891a23c9686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733235346956 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cfd7885b434dbdbb6c8924516d4bed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733235346956 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811fa6344977452a92dfd98b105e2a5a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733235347610 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 371cbd2836734b9d83d7fc93174cb3f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733235347610 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b131f8aa067a4fd7bb98e2cf7b9b34dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733235348286 2024-12-03T14:15:50,962 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 753e84cba117493ba0050ccddfa78bc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733235348286 2024-12-03T14:15:50,963 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ab37980f4d98426ea92eee94363dee3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:50,963 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd742f2ad0524acf96980f88a7922459, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:50,969 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:50,970 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/b7e23c5a91bb4123be35032d57982ca4 is 50, key is test_row_0/B:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:50,970 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:50,971 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/8e1497654d4847d3bffac1db90d2098c is 50, key is test_row_0/A:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:50,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742276_1452 (size=12173) 2024-12-03T14:15:50,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742277_1453 (size=12173) 2024-12-03T14:15:50,982 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/8e1497654d4847d3bffac1db90d2098c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/8e1497654d4847d3bffac1db90d2098c 2024-12-03T14:15:50,985 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 8e1497654d4847d3bffac1db90d2098c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:50,985 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:50,985 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=11, startTime=1733235350958; duration=0sec 2024-12-03T14:15:50,985 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:50,986 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:15:50,986 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:15:50,987 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:15:50,987 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:15:50,987 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:50,987 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=56.3 K 2024-12-03T14:15:50,989 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355116a291b94c1e9e9420680fe698bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733235346906 2024-12-03T14:15:50,989 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bae927f39ba4858b7e85182230cad4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733235346956 2024-12-03T14:15:50,990 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52900265a7e4cb3a2f4913ee2427f4c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733235347610 2024-12-03T14:15:50,990 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f51aa93f43b4f68b4bccf0d0376f37a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733235348286 2024-12-03T14:15:50,990 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e61d2e00f9784e84bd2c2536ac6da291, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:50,997 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:50,998 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/87a8d74b44404f68a075c999946fe971 is 50, key is test_row_0/C:col10/1733235350108/Put/seqid=0 2024-12-03T14:15:51,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742278_1454 (size=12173) 2024-12-03T14:15:51,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:51,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:51,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3a40ebd7b53a47d19b7be533cc92fb3f is 50, key is test_row_0/A:col10/1733235350207/Put/seqid=0 2024-12-03T14:15:51,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742279_1455 (size=12151) 2024-12-03T14:15:51,102 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3a40ebd7b53a47d19b7be533cc92fb3f 2024-12-03T14:15:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/71197e87b76340d7a19229aabf4ef539 is 50, key is test_row_0/B:col10/1733235350207/Put/seqid=0 2024-12-03T14:15:51,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742280_1456 (size=12151) 2024-12-03T14:15:51,377 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/b7e23c5a91bb4123be35032d57982ca4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/b7e23c5a91bb4123be35032d57982ca4 2024-12-03T14:15:51,381 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into b7e23c5a91bb4123be35032d57982ca4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:51,381 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:51,381 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=11, startTime=1733235350959; duration=0sec 2024-12-03T14:15:51,381 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:51,381 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:15:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:51,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:51,407 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/87a8d74b44404f68a075c999946fe971 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/87a8d74b44404f68a075c999946fe971 2024-12-03T14:15:51,412 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into 87a8d74b44404f68a075c999946fe971(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:51,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:51,412 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=11, startTime=1733235350959; duration=0sec 2024-12-03T14:15:51,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:51,412 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:15:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235411408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235411409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235411413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:51,513 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/71197e87b76340d7a19229aabf4ef539 2024-12-03T14:15:51,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d36ef5944ca84663bb4e55215a884734 is 50, key is test_row_0/C:col10/1733235350207/Put/seqid=0 2024-12-03T14:15:51,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742281_1457 (size=12151) 2024-12-03T14:15:51,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235411526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235411736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235411736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235411746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:51,927 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d36ef5944ca84663bb4e55215a884734 2024-12-03T14:15:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3a40ebd7b53a47d19b7be533cc92fb3f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f 2024-12-03T14:15:51,933 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:15:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/71197e87b76340d7a19229aabf4ef539 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539 2024-12-03T14:15:51,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:15:51,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d36ef5944ca84663bb4e55215a884734 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734 2024-12-03T14:15:51,940 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734, entries=150, sequenceid=138, filesize=11.9 K 2024-12-03T14:15:51,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 846ms, sequenceid=138, compaction requested=false 2024-12-03T14:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T14:15:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T14:15:51,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T14:15:51,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-12-03T14:15:51,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6170 sec 2024-12-03T14:15:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:52,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:52,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/357a197aba004ea6a7064eed1b090d16 is 50, key is test_row_0/A:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742282_1458 (size=14541) 2024-12-03T14:15:52,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/357a197aba004ea6a7064eed1b090d16 2024-12-03T14:15:52,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/db344e1288e44a18bc3ff46f9460f107 is 50, key is test_row_0/B:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742283_1459 (size=12151) 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235352066, max timestamp=1733235352122Num uniques 859; 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235352066, max timestamp=1733235352122Num uniques 859; 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235352087, max timestamp=1733235352122Num uniques 859; 2024-12-03T14:15:52,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:15:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235412157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235412158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235412163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235412276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235412276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235412276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/db344e1288e44a18bc3ff46f9460f107 2024-12-03T14:15:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T14:15:52,477 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:52,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T14:15:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T14:15:52,478 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:52,479 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:52,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:52,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4adcbebf0682470b8058ccb68f62346a is 50, key is test_row_0/C:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235412486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235412486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235412486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742284_1460 (size=12151) 2024-12-03T14:15:52,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4adcbebf0682470b8058ccb68f62346a 2024-12-03T14:15:52,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/357a197aba004ea6a7064eed1b090d16 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16 2024-12-03T14:15:52,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16, entries=200, sequenceid=151, filesize=14.2 K 2024-12-03T14:15:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235412516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235412516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,525 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,525 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/db344e1288e44a18bc3ff46f9460f107 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107 2024-12-03T14:15:52,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107, entries=150, sequenceid=151, filesize=11.9 K 2024-12-03T14:15:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4adcbebf0682470b8058ccb68f62346a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a 2024-12-03T14:15:52,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a, entries=150, sequenceid=151, filesize=11.9 K 2024-12-03T14:15:52,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 494ms, sequenceid=151, compaction requested=true 2024-12-03T14:15:52,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:52,543 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:52,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:52,544 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:52,544 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:52,544 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:15:52,544 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:52,544 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/8e1497654d4847d3bffac1db90d2098c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=38.0 K 2024-12-03T14:15:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e1497654d4847d3bffac1db90d2098c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:15:52,545 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:52,545 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/b7e23c5a91bb4123be35032d57982ca4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=35.6 K 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e23c5a91bb4123be35032d57982ca4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a40ebd7b53a47d19b7be533cc92fb3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235350207 2024-12-03T14:15:52,545 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 71197e87b76340d7a19229aabf4ef539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235350207 2024-12-03T14:15:52,546 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357a197aba004ea6a7064eed1b090d16, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351399 2024-12-03T14:15:52,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:52,546 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting db344e1288e44a18bc3ff46f9460f107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351408 2024-12-03T14:15:52,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:52,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:52,574 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:52,575 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f7656ca8959247a6b332ac451a86357e is 50, key is test_row_0/B:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,581 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:52,581 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/08e6d4f9feb14d4185ac2e7b7a04bb63 is 50, key is test_row_0/A:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T14:15:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742285_1461 (size=12425) 2024-12-03T14:15:52,602 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f7656ca8959247a6b332ac451a86357e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f7656ca8959247a6b332ac451a86357e 2024-12-03T14:15:52,607 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into f7656ca8959247a6b332ac451a86357e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:52,607 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:52,607 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235352544; duration=0sec 2024-12-03T14:15:52,607 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:52,607 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:15:52,607 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:52,613 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:52,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:15:52,614 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:52,614 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/87a8d74b44404f68a075c999946fe971, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=35.6 K 2024-12-03T14:15:52,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a8d74b44404f68a075c999946fe971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733235349462 2024-12-03T14:15:52,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d36ef5944ca84663bb4e55215a884734, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733235350207 2024-12-03T14:15:52,615 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adcbebf0682470b8058ccb68f62346a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351408 2024-12-03T14:15:52,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742286_1462 (size=12425) 2024-12-03T14:15:52,626 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:52,627 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f8bd1e5ffb814741b54ea524974e44db is 50, key is test_row_0/C:col10/1733235351411/Put/seqid=0 2024-12-03T14:15:52,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T14:15:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:52,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-12-03T14:15:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:15:52,633 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/08e6d4f9feb14d4185ac2e7b7a04bb63 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/08e6d4f9feb14d4185ac2e7b7a04bb63 2024-12-03T14:15:52,640 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 08e6d4f9feb14d4185ac2e7b7a04bb63(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:52,642 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:52,642 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235352543; duration=0sec 2024-12-03T14:15:52,642 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:52,642 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:15:52,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc05df2727194c33bd8c99587e81038e is 50, key is test_row_0/A:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:52,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742287_1463 (size=12425) 2024-12-03T14:15:52,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f8bd1e5ffb814741b54ea524974e44db as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8bd1e5ffb814741b54ea524974e44db 2024-12-03T14:15:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742288_1464 (size=12151) 2024-12-03T14:15:52,683 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into f8bd1e5ffb814741b54ea524974e44db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:52,683 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc05df2727194c33bd8c99587e81038e 2024-12-03T14:15:52,683 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:52,683 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=13, startTime=1733235352546; duration=0sec 2024-12-03T14:15:52,683 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:52,683 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:15:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6eecb14595d43f2810d43f329ec812e is 50, key is test_row_0/B:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:52,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742289_1465 (size=12151) 2024-12-03T14:15:52,698 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6eecb14595d43f2810d43f329ec812e 2024-12-03T14:15:52,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/42bcc2f7694b40cf8fb6a89a387d899a is 50, key is test_row_0/C:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742290_1466 (size=12151) 2024-12-03T14:15:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T14:15:52,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235412822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235412835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235412835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235412946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235412946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235412957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T14:15:53,118 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/42bcc2f7694b40cf8fb6a89a387d899a 2024-12-03T14:15:53,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc05df2727194c33bd8c99587e81038e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e 2024-12-03T14:15:53,124 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T14:15:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e6eecb14595d43f2810d43f329ec812e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e 2024-12-03T14:15:53,128 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T14:15:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/42bcc2f7694b40cf8fb6a89a387d899a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a 2024-12-03T14:15:53,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T14:15:53,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 500ms, sequenceid=187, compaction requested=false 2024-12-03T14:15:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T14:15:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T14:15:53,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T14:15:53,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 654 msec 2024-12-03T14:15:53,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 658 msec 2024-12-03T14:15:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:53,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:53,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/7c7ba999193e4d8592485798467a1e5c is 50, key is test_row_1/A:col10/1733235353158/Put/seqid=0 2024-12-03T14:15:53,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742291_1467 (size=12147) 2024-12-03T14:15:53,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235413195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235413197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235413202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235413306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235413315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235413316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235413516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235413526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235413527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/7c7ba999193e4d8592485798467a1e5c 2024-12-03T14:15:53,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/7c82d1704bf54e1a9a390f3f50904f66 is 50, key is test_row_1/B:col10/1733235353158/Put/seqid=0 2024-12-03T14:15:53,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742292_1468 (size=9757) 2024-12-03T14:15:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T14:15:53,616 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:53,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T14:15:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:53,618 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:53,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:53,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:53,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:53,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235413826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235413846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235413846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:53,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:53,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:53,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:53,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/7c82d1704bf54e1a9a390f3f50904f66 2024-12-03T14:15:53,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f5c4f11ac03d49a0ac35b1c254600b00 is 50, key is test_row_1/C:col10/1733235353158/Put/seqid=0 2024-12-03T14:15:53,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742293_1469 (size=9757) 2024-12-03T14:15:54,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:54,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235414336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:54,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235414356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235414356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:54,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:54,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:54,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f5c4f11ac03d49a0ac35b1c254600b00 2024-12-03T14:15:54,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/7c7ba999193e4d8592485798467a1e5c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c 2024-12-03T14:15:54,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c, entries=150, sequenceid=202, filesize=11.9 K 2024-12-03T14:15:54,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/7c82d1704bf54e1a9a390f3f50904f66 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66 2024-12-03T14:15:54,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66, entries=100, sequenceid=202, filesize=9.5 K 2024-12-03T14:15:54,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f5c4f11ac03d49a0ac35b1c254600b00 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00 2024-12-03T14:15:54,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00, entries=100, sequenceid=202, filesize=9.5 K 2024-12-03T14:15:54,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1247ms, sequenceid=202, compaction requested=true 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:54,407 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:54,407 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:54,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36723 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:15:54,408 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,408 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,408 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/08e6d4f9feb14d4185ac2e7b7a04bb63, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=35.9 K 2024-12-03T14:15:54,408 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f7656ca8959247a6b332ac451a86357e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=33.5 K 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08e6d4f9feb14d4185ac2e7b7a04bb63, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351408 2024-12-03T14:15:54,408 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f7656ca8959247a6b332ac451a86357e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351408 2024-12-03T14:15:54,409 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e6eecb14595d43f2810d43f329ec812e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733235352156 2024-12-03T14:15:54,409 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c82d1704bf54e1a9a390f3f50904f66, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235353158 2024-12-03T14:15:54,409 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc05df2727194c33bd8c99587e81038e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733235352156 2024-12-03T14:15:54,409 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7ba999193e4d8592485798467a1e5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235352833 2024-12-03T14:15:54,418 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:54,418 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/6c2f72bc26b44b81a8d9bdce620133aa is 50, key is test_row_0/A:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:54,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742294_1470 (size=12527) 2024-12-03T14:15:54,429 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:54,429 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e66a172f600e4a1ebed4e3ee9a5d8e52 is 50, key is test_row_0/B:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:54,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742295_1471 (size=12527) 2024-12-03T14:15:54,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,535 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/1ff9335af3c643ffb7030935c0a66467 is 50, key is test_row_0/A:col10/1733235353194/Put/seqid=0 2024-12-03T14:15:54,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742296_1472 (size=12151) 2024-12-03T14:15:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:54,833 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/6c2f72bc26b44b81a8d9bdce620133aa as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/6c2f72bc26b44b81a8d9bdce620133aa 2024-12-03T14:15:54,836 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 6c2f72bc26b44b81a8d9bdce620133aa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:54,836 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:54,836 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235354407; duration=0sec 2024-12-03T14:15:54,837 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:54,837 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:15:54,837 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:54,837 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:54,837 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:15:54,837 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:54,838 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8bd1e5ffb814741b54ea524974e44db, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=33.5 K 2024-12-03T14:15:54,838 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8bd1e5ffb814741b54ea524974e44db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733235351408 2024-12-03T14:15:54,838 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bcc2f7694b40cf8fb6a89a387d899a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733235352156 2024-12-03T14:15:54,839 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c4f11ac03d49a0ac35b1c254600b00, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235353158 2024-12-03T14:15:54,842 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/e66a172f600e4a1ebed4e3ee9a5d8e52 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e66a172f600e4a1ebed4e3ee9a5d8e52 2024-12-03T14:15:54,854 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:54,854 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fadd25f0102e433395a1249cbcc78bc6 is 50, key is test_row_0/C:col10/1733235352162/Put/seqid=0 2024-12-03T14:15:54,861 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into e66a172f600e4a1ebed4e3ee9a5d8e52(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:54,861 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:54,861 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235354407; duration=0sec 2024-12-03T14:15:54,861 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:54,861 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:15:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742297_1473 (size=12527) 2024-12-03T14:15:54,942 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/1ff9335af3c643ffb7030935c0a66467 2024-12-03T14:15:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/74ccbbbea1224b2783b17d310cb23bd8 is 50, key is test_row_0/B:col10/1733235353194/Put/seqid=0 2024-12-03T14:15:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742298_1474 (size=12151) 2024-12-03T14:15:54,957 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/74ccbbbea1224b2783b17d310cb23bd8 2024-12-03T14:15:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 is 50, key is test_row_0/C:col10/1733235353194/Put/seqid=0 2024-12-03T14:15:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742299_1475 (size=12151) 2024-12-03T14:15:55,267 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fadd25f0102e433395a1249cbcc78bc6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fadd25f0102e433395a1249cbcc78bc6 2024-12-03T14:15:55,270 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into fadd25f0102e433395a1249cbcc78bc6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:55,270 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:55,270 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=13, startTime=1733235354407; duration=0sec 2024-12-03T14:15:55,271 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:55,271 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:15:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:55,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:55,365 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 2024-12-03T14:15:55,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/1ff9335af3c643ffb7030935c0a66467 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467 2024-12-03T14:15:55,372 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T14:15:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/74ccbbbea1224b2783b17d310cb23bd8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8 2024-12-03T14:15:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T14:15:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 2024-12-03T14:15:55,388 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T14:15:55,389 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=60.38 KB/61830 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 854ms, sequenceid=221, compaction requested=false 2024-12-03T14:15:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:55,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T14:15:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T14:15:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:55,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:15:55,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:55,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:55,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:55,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:55,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:55,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:55,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T14:15:55,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-12-03T14:15:55,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7760 sec 2024-12-03T14:15:55,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3cd495d9cd6b4ba58d85272f68de0770 is 50, key is test_row_0/A:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742300_1476 (size=16931) 2024-12-03T14:15:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235415423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235415424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235415425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235415536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235415536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235415536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235415747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T14:15:55,766 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:55,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T14:15:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:55,768 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:55,769 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:55,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:15:55,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3cd495d9cd6b4ba58d85272f68de0770 2024-12-03T14:15:55,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2bd58e4167194ec78863a2b5fbb3462e is 50, key is test_row_0/B:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:55,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742301_1477 (size=12151) 2024-12-03T14:15:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:55,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:55,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235416056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235416056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235416066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:56,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2bd58e4167194ec78863a2b5fbb3462e 2024-12-03T14:15:56,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:56,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:56,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4a49d63d2c26454d9140a35e76fdf7c0 is 50, key is test_row_0/C:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:56,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742302_1478 (size=12151) 2024-12-03T14:15:56,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:56,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:56,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:56,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:15:56,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235416536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,538 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235416566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,571 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_1' in e0c0e6a2f069c0cb795f0cc0acfdf529 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235416576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235416576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235416576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:56,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4a49d63d2c26454d9140a35e76fdf7c0 2024-12-03T14:15:56,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/3cd495d9cd6b4ba58d85272f68de0770 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770 2024-12-03T14:15:56,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770, entries=250, sequenceid=239, filesize=16.5 K 2024-12-03T14:15:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2bd58e4167194ec78863a2b5fbb3462e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e 2024-12-03T14:15:56,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T14:15:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4a49d63d2c26454d9140a35e76fdf7c0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0 2024-12-03T14:15:56,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0, entries=150, sequenceid=239, filesize=11.9 K 2024-12-03T14:15:56,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1258ms, sequenceid=239, compaction requested=true 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:56,648 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:56,648 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:15:56,649 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,649 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,649 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e66a172f600e4a1ebed4e3ee9a5d8e52, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.0 K 2024-12-03T14:15:56,649 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/6c2f72bc26b44b81a8d9bdce620133aa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=40.6 K 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e66a172f600e4a1ebed4e3ee9a5d8e52, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235352162 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c2f72bc26b44b81a8d9bdce620133aa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235352162 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff9335af3c643ffb7030935c0a66467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733235353192 2024-12-03T14:15:56,649 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ccbbbea1224b2783b17d310cb23bd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733235353192 2024-12-03T14:15:56,650 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cd495d9cd6b4ba58d85272f68de0770, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355363 2024-12-03T14:15:56,650 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd58e4167194ec78863a2b5fbb3462e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355381 2024-12-03T14:15:56,656 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:56,656 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:56,657 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/37ea5247ac0a4df5bbba878892818355 is 50, key is test_row_0/B:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:56,657 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/7f7eeef3bd8f4417b2f8863135c1955d is 50, key is test_row_0/A:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:56,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742304_1480 (size=12629) 2024-12-03T14:15:56,673 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/7f7eeef3bd8f4417b2f8863135c1955d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7f7eeef3bd8f4417b2f8863135c1955d 2024-12-03T14:15:56,677 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 7f7eeef3bd8f4417b2f8863135c1955d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:56,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:56,677 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235356648; duration=0sec 2024-12-03T14:15:56,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:56,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:15:56,677 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:56,678 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:56,678 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:15:56,678 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,678 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fadd25f0102e433395a1249cbcc78bc6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.0 K 2024-12-03T14:15:56,678 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadd25f0102e433395a1249cbcc78bc6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733235352162 2024-12-03T14:15:56,679 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting d05e13c5c3fe44ffb40cd0dad5a4a0c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733235353192 2024-12-03T14:15:56,679 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a49d63d2c26454d9140a35e76fdf7c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355381 2024-12-03T14:15:56,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742303_1479 (size=12629) 2024-12-03T14:15:56,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T14:15:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:56,684 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T14:15:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:56,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:56,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/ddc4966ec38347a4bc04b33bb5fa93f8 is 50, key is test_row_0/A:col10/1733235355424/Put/seqid=0 2024-12-03T14:15:56,695 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:56,696 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/6fdad1107a21445b88048acf7173eace is 50, key is test_row_0/C:col10/1733235355381/Put/seqid=0 2024-12-03T14:15:56,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742305_1481 (size=12151) 2024-12-03T14:15:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742306_1482 (size=12629) 2024-12-03T14:15:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:57,086 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/37ea5247ac0a4df5bbba878892818355 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/37ea5247ac0a4df5bbba878892818355 2024-12-03T14:15:57,089 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into 37ea5247ac0a4df5bbba878892818355(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:57,089 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:57,089 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235356648; duration=0sec 2024-12-03T14:15:57,089 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:57,089 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:15:57,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/ddc4966ec38347a4bc04b33bb5fa93f8 2024-12-03T14:15:57,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2ba1d629ec584e5393656711147f72e1 is 50, key is test_row_0/B:col10/1733235355424/Put/seqid=0 2024-12-03T14:15:57,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742307_1483 (size=12151) 2024-12-03T14:15:57,125 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/6fdad1107a21445b88048acf7173eace as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/6fdad1107a21445b88048acf7173eace 2024-12-03T14:15:57,130 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into 6fdad1107a21445b88048acf7173eace(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:57,130 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:57,130 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=13, startTime=1733235356648; duration=0sec 2024-12-03T14:15:57,130 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:57,130 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:15:57,511 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2ba1d629ec584e5393656711147f72e1 2024-12-03T14:15:57,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df13160206114db3b0b58803379be247 is 50, key is test_row_0/C:col10/1733235355424/Put/seqid=0 2024-12-03T14:15:57,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742308_1484 (size=12151) 2024-12-03T14:15:57,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:57,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235417616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235417617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235417617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235417726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235417736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235417736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:57,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df13160206114db3b0b58803379be247 2024-12-03T14:15:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/ddc4966ec38347a4bc04b33bb5fa93f8 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8 2024-12-03T14:15:57,927 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8, entries=150, sequenceid=257, filesize=11.9 K 2024-12-03T14:15:57,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2ba1d629ec584e5393656711147f72e1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1 2024-12-03T14:15:57,932 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1, entries=150, sequenceid=257, filesize=11.9 K 2024-12-03T14:15:57,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df13160206114db3b0b58803379be247 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247 2024-12-03T14:15:57,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247, entries=150, sequenceid=257, filesize=11.9 K 2024-12-03T14:15:57,938 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1254ms, sequenceid=257, compaction requested=false 2024-12-03T14:15:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T14:15:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T14:15:57,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:15:57,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:15:57,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:15:57,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T14:15:57,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-12-03T14:15:57,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1750 sec 2024-12-03T14:15:57,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/cab0b6e5a4914d0892bb84b083ffcda5 is 50, key is test_row_0/A:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:57,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742309_1485 (size=14741) 2024-12-03T14:15:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235417976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235417980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235417981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235418085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235418086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235418096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235418296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235418296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235418316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/cab0b6e5a4914d0892bb84b083ffcda5 2024-12-03T14:15:58,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d76eafe9a7a24dcb9909372ccf4f9e82 is 50, key is test_row_0/B:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742310_1486 (size=12301) 2024-12-03T14:15:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235418606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235418606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235418626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:58,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d76eafe9a7a24dcb9909372ccf4f9e82 2024-12-03T14:15:58,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fe224c8c42cc4a8cb86ba27390c37815 is 50, key is test_row_0/C:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742311_1487 (size=12301) 2024-12-03T14:15:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235419116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235419126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:15:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235419136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:15:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:15:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:15:59,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fe224c8c42cc4a8cb86ba27390c37815 2024-12-03T14:15:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/cab0b6e5a4914d0892bb84b083ffcda5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5 2024-12-03T14:15:59,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5, entries=200, sequenceid=275, filesize=14.4 K 2024-12-03T14:15:59,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d76eafe9a7a24dcb9909372ccf4f9e82 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82 2024-12-03T14:15:59,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82, entries=150, sequenceid=275, filesize=12.0 K 2024-12-03T14:15:59,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fe224c8c42cc4a8cb86ba27390c37815 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815 2024-12-03T14:15:59,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815, entries=150, sequenceid=275, filesize=12.0 K 2024-12-03T14:15:59,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1252ms, sequenceid=275, compaction requested=true 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:59,192 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:59,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:15:59,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:59,194 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:59,194 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:15:59,194 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:59,194 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7f7eeef3bd8f4417b2f8863135c1955d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=38.6 K 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7eeef3bd8f4417b2f8863135c1955d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355381 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:15:59,195 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc4966ec38347a4bc04b33bb5fa93f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733235355422 2024-12-03T14:15:59,195 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/37ea5247ac0a4df5bbba878892818355, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.2 K 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab0b6e5a4914d0892bb84b083ffcda5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357614 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ea5247ac0a4df5bbba878892818355, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355381 2024-12-03T14:15:59,195 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba1d629ec584e5393656711147f72e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733235355422 2024-12-03T14:15:59,196 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d76eafe9a7a24dcb9909372ccf4f9e82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357616 2024-12-03T14:15:59,201 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:59,201 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#412 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:59,202 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/55d154e1c51441ad914afa8d28eabb15 is 50, key is test_row_0/B:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:59,202 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/53521c7542034bb68984bb48695d4103 is 50, key is test_row_0/A:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:59,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742313_1489 (size=12881) 2024-12-03T14:15:59,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742312_1488 (size=12881) 2024-12-03T14:15:59,608 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/55d154e1c51441ad914afa8d28eabb15 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/55d154e1c51441ad914afa8d28eabb15 2024-12-03T14:15:59,610 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/53521c7542034bb68984bb48695d4103 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/53521c7542034bb68984bb48695d4103 2024-12-03T14:15:59,613 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into 55d154e1c51441ad914afa8d28eabb15(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:59,613 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:59,613 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235359192; duration=0sec 2024-12-03T14:15:59,613 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:15:59,613 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:15:59,613 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:15:59,614 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:15:59,614 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/6fdad1107a21445b88048acf7173eace, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.2 K 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fdad1107a21445b88048acf7173eace, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733235355381 2024-12-03T14:15:59,614 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 53521c7542034bb68984bb48695d4103(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:59,614 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235359192; duration=0sec 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:59,614 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:15:59,615 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting df13160206114db3b0b58803379be247, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733235355422 2024-12-03T14:15:59,615 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting fe224c8c42cc4a8cb86ba27390c37815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357616 2024-12-03T14:15:59,620 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:15:59,621 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/b55bba747b4a4f6088bb6b83aca22d84 is 50, key is test_row_0/C:col10/1733235357938/Put/seqid=0 2024-12-03T14:15:59,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742314_1490 (size=12881) 2024-12-03T14:15:59,628 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/b55bba747b4a4f6088bb6b83aca22d84 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b55bba747b4a4f6088bb6b83aca22d84 2024-12-03T14:15:59,633 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into b55bba747b4a4f6088bb6b83aca22d84(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:15:59,633 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:15:59,633 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=13, startTime=1733235359192; duration=0sec 2024-12-03T14:15:59,633 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:15:59,633 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:15:59,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T14:15:59,937 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:15:59,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:15:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T14:15:59,939 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:15:59,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T14:15:59,940 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:15:59,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T14:16:00,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T14:16:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:00,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:16:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/a812d4551ee846e09a9170d0ccd5c79c is 50, key is test_row_0/A:col10/1733235357979/Put/seqid=0 2024-12-03T14:16:00,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742315_1491 (size=12301) 2024-12-03T14:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:00,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235420173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235420174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235420179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T14:16:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235420285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235420296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235420296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235420496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235420505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235420506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,522 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/a812d4551ee846e09a9170d0ccd5c79c 2024-12-03T14:16:00,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8f63b3ad4d554079aec970eeceb628a3 is 50, key is test_row_0/B:col10/1733235357979/Put/seqid=0 2024-12-03T14:16:00,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742316_1492 (size=12301) 2024-12-03T14:16:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T14:16:00,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235420806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235420816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235420816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:00,930 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8f63b3ad4d554079aec970eeceb628a3 2024-12-03T14:16:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/46bb13408465487caf1ce81cc1c23ec5 is 50, key is test_row_0/C:col10/1733235357979/Put/seqid=0 2024-12-03T14:16:00,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742317_1493 (size=12301) 2024-12-03T14:16:00,941 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/46bb13408465487caf1ce81cc1c23ec5 2024-12-03T14:16:00,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/a812d4551ee846e09a9170d0ccd5c79c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c 2024-12-03T14:16:00,951 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:16:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8f63b3ad4d554079aec970eeceb628a3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3 2024-12-03T14:16:00,955 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:16:00,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/46bb13408465487caf1ce81cc1c23ec5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5 2024-12-03T14:16:00,960 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5, entries=150, sequenceid=294, filesize=12.0 K 2024-12-03T14:16:00,961 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 862ms, sequenceid=294, compaction requested=false 2024-12-03T14:16:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T14:16:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T14:16:00,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T14:16:00,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-12-03T14:16:00,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0270 sec 2024-12-03T14:16:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T14:16:01,079 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:01,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T14:16:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T14:16:01,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:01,081 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:01,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T14:16:01,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T14:16:01,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:01,233 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:01,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/37b81440e4d24ecda239843d9d3e62f7 is 50, key is test_row_0/A:col10/1733235360163/Put/seqid=0 2024-12-03T14:16:01,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742318_1494 (size=12301) 2024-12-03T14:16:01,241 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/37b81440e4d24ecda239843d9d3e62f7 2024-12-03T14:16:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ed9c9693a0d2450db5e4b4b893f7409e is 50, key is test_row_0/B:col10/1733235360163/Put/seqid=0 2024-12-03T14:16:01,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742319_1495 (size=12301) 2024-12-03T14:16:01,251 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ed9c9693a0d2450db5e4b4b893f7409e 2024-12-03T14:16:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9bdf23c91c59401bb6d76906e3f1fcc7 is 50, key is test_row_0/C:col10/1733235360163/Put/seqid=0 2024-12-03T14:16:01,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742320_1496 (size=12301) 2024-12-03T14:16:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:01,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:01,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235421371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235421373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235421373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T14:16:01,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235421486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235421486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235421486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,673 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9bdf23c91c59401bb6d76906e3f1fcc7 2024-12-03T14:16:01,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/37b81440e4d24ecda239843d9d3e62f7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7 2024-12-03T14:16:01,679 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7, entries=150, sequenceid=310, filesize=12.0 K 2024-12-03T14:16:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/ed9c9693a0d2450db5e4b4b893f7409e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e 2024-12-03T14:16:01,683 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e, entries=150, sequenceid=310, filesize=12.0 K 2024-12-03T14:16:01,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9bdf23c91c59401bb6d76906e3f1fcc7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7 2024-12-03T14:16:01,686 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7, entries=150, sequenceid=310, filesize=12.0 K 2024-12-03T14:16:01,687 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 454ms, sequenceid=310, compaction requested=true 2024-12-03T14:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T14:16:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T14:16:01,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T14:16:01,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 607 msec 2024-12-03T14:16:01,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 611 msec 2024-12-03T14:16:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:01,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:01,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T14:16:01,706 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:01,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T14:16:01,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:01,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:01,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:01,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/097d895393e44633b467f8bc19c9319f is 50, key is test_row_0/A:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:01,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742321_1497 (size=17181) 2024-12-03T14:16:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235421721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235421723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235421728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:01,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235421836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235421836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235421836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:01,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:01,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:02,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235422046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235422046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235422046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/097d895393e44633b467f8bc19c9319f 2024-12-03T14:16:02,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f049c458feae4e33a4b1dfc70564a4ff is 50, key is test_row_0/B:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:02,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742322_1498 (size=12301) 2024-12-03T14:16:02,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235422356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235422356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235422356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f049c458feae4e33a4b1dfc70564a4ff 2024-12-03T14:16:02,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9f87147a052144e6927e546cf1263387 is 50, key is test_row_0/C:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742323_1499 (size=12301) 2024-12-03T14:16:02,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:02,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235422866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235422866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235422866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:02,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:02,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:02,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9f87147a052144e6927e546cf1263387 2024-12-03T14:16:02,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/097d895393e44633b467f8bc19c9319f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f 2024-12-03T14:16:02,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f, entries=250, sequenceid=329, filesize=16.8 K 2024-12-03T14:16:02,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f049c458feae4e33a4b1dfc70564a4ff as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff 2024-12-03T14:16:02,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff, entries=150, sequenceid=329, filesize=12.0 K 2024-12-03T14:16:02,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/9f87147a052144e6927e546cf1263387 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387 2024-12-03T14:16:02,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387, entries=150, sequenceid=329, filesize=12.0 K 2024-12-03T14:16:02,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1251ms, sequenceid=329, compaction requested=true 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:02,955 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:02,955 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:02,956 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:02,956 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:02,956 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:16:02,956 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:16:02,956 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,956 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:02,956 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/53521c7542034bb68984bb48695d4103, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=53.4 K 2024-12-03T14:16:02,956 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/55d154e1c51441ad914afa8d28eabb15, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=48.6 K 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53521c7542034bb68984bb48695d4103, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357616 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d154e1c51441ad914afa8d28eabb15, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357616 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f63b3ad4d554079aec970eeceb628a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235357957 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a812d4551ee846e09a9170d0ccd5c79c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235357957 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting ed9c9693a0d2450db5e4b4b893f7409e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733235360163 2024-12-03T14:16:02,957 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b81440e4d24ecda239843d9d3e62f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733235360163 2024-12-03T14:16:02,958 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 097d895393e44633b467f8bc19c9319f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361348 2024-12-03T14:16:02,958 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f049c458feae4e33a4b1dfc70564a4ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361371 2024-12-03T14:16:02,970 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:02,971 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f36a68b4fdd84418a4a3b9e10a563de6 is 50, key is test_row_0/B:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:02,971 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:02,972 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/35afb14f6dc044eeb64cddd8430a2792 is 50, key is test_row_0/A:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742324_1500 (size=13017) 2024-12-03T14:16:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742325_1501 (size=13017) 2024-12-03T14:16:03,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:03,080 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc8031cd8b79426d99d4bd3380a4af31 is 50, key is test_row_0/A:col10/1733235361719/Put/seqid=0 2024-12-03T14:16:03,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742326_1502 (size=12301) 2024-12-03T14:16:03,392 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/35afb14f6dc044eeb64cddd8430a2792 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/35afb14f6dc044eeb64cddd8430a2792 2024-12-03T14:16:03,392 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/f36a68b4fdd84418a4a3b9e10a563de6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f36a68b4fdd84418a4a3b9e10a563de6 2024-12-03T14:16:03,400 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into f36a68b4fdd84418a4a3b9e10a563de6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:03,400 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 35afb14f6dc044eeb64cddd8430a2792(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:03,401 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=12, startTime=1733235362955; duration=0sec 2024-12-03T14:16:03,401 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=12, startTime=1733235362955; duration=0sec 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:03,401 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:03,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:03,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:16:03,403 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:03,403 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b55bba747b4a4f6088bb6b83aca22d84, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=48.6 K 2024-12-03T14:16:03,404 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting b55bba747b4a4f6088bb6b83aca22d84, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733235357616 2024-12-03T14:16:03,404 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bb13408465487caf1ce81cc1c23ec5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733235357957 2024-12-03T14:16:03,405 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bdf23c91c59401bb6d76906e3f1fcc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733235360163 2024-12-03T14:16:03,406 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f87147a052144e6927e546cf1263387, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361371 2024-12-03T14:16:03,413 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:03,413 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f8c3cd94205b411f9d3866aa0b4688e0 is 50, key is test_row_0/C:col10/1733235361703/Put/seqid=0 2024-12-03T14:16:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742327_1503 (size=13017) 2024-12-03T14:16:03,487 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc8031cd8b79426d99d4bd3380a4af31 2024-12-03T14:16:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/86bad225264a47158fe2bb2af346e293 is 50, key is test_row_0/B:col10/1733235361719/Put/seqid=0 2024-12-03T14:16:03,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742328_1504 (size=12301) 2024-12-03T14:16:03,823 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/f8c3cd94205b411f9d3866aa0b4688e0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8c3cd94205b411f9d3866aa0b4688e0 2024-12-03T14:16:03,831 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into f8c3cd94205b411f9d3866aa0b4688e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:03,831 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:03,831 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=12, startTime=1733235362955; duration=0sec 2024-12-03T14:16:03,831 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:03,831 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:03,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:03,895 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/86bad225264a47158fe2bb2af346e293 2024-12-03T14:16:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fa90006efb094fcea194ee13aa05aee6 is 50, key is test_row_0/C:col10/1733235361719/Put/seqid=0 2024-12-03T14:16:03,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742329_1505 (size=12301) 2024-12-03T14:16:03,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235423909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:03,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235423909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:03,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235423910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235424016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235424026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235424026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235424226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235424236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235424236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,305 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fa90006efb094fcea194ee13aa05aee6 2024-12-03T14:16:04,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/fc8031cd8b79426d99d4bd3380a4af31 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31 2024-12-03T14:16:04,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T14:16:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/86bad225264a47158fe2bb2af346e293 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293 2024-12-03T14:16:04,319 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T14:16:04,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/fa90006efb094fcea194ee13aa05aee6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6 2024-12-03T14:16:04,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T14:16:04,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1243ms, sequenceid=342, compaction requested=false 2024-12-03T14:16:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T14:16:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T14:16:04,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T14:16:04,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6170 sec 2024-12-03T14:16:04,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.6210 sec 2024-12-03T14:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:04,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:04,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:04,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c01dd381a3ec4aef80af32a8f550e290 is 50, key is test_row_0/A:col10/1733235363907/Put/seqid=0 2024-12-03T14:16:04,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742330_1506 (size=14741) 2024-12-03T14:16:04,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c01dd381a3ec4aef80af32a8f550e290 2024-12-03T14:16:04,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2e341302e91d4fcd8fd586369b190e72 is 50, key is test_row_0/B:col10/1733235363907/Put/seqid=0 2024-12-03T14:16:04,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742331_1507 (size=12301) 2024-12-03T14:16:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235424556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235424561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235424562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235424666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235424666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235424675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235424876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235424876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235424886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:04,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2e341302e91d4fcd8fd586369b190e72 2024-12-03T14:16:04,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/0938c2b546b54021b0f7dbb887df7769 is 50, key is test_row_0/C:col10/1733235363907/Put/seqid=0 2024-12-03T14:16:04,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742332_1508 (size=12301) 2024-12-03T14:16:05,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235425185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235425186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235425196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/0938c2b546b54021b0f7dbb887df7769 2024-12-03T14:16:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/c01dd381a3ec4aef80af32a8f550e290 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290 2024-12-03T14:16:05,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290, entries=200, sequenceid=365, filesize=14.4 K 2024-12-03T14:16:05,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/2e341302e91d4fcd8fd586369b190e72 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72 2024-12-03T14:16:05,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T14:16:05,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/0938c2b546b54021b0f7dbb887df7769 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769 2024-12-03T14:16:05,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769, entries=150, sequenceid=365, filesize=12.0 K 2024-12-03T14:16:05,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 838ms, sequenceid=365, compaction requested=true 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:05,378 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:05,378 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:05,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:05,378 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:05,378 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:16:05,379 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:05,379 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:05,379 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f36a68b4fdd84418a4a3b9e10a563de6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.7 K 2024-12-03T14:16:05,379 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/35afb14f6dc044eeb64cddd8430a2792, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=39.1 K 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f36a68b4fdd84418a4a3b9e10a563de6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361371 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35afb14f6dc044eeb64cddd8430a2792, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361371 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 86bad225264a47158fe2bb2af346e293, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235361719 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc8031cd8b79426d99d4bd3380a4af31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235361719 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01dd381a3ec4aef80af32a8f550e290, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235363907 2024-12-03T14:16:05,379 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e341302e91d4fcd8fd586369b190e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235363907 2024-12-03T14:16:05,385 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:05,385 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/92dea45e414e437db85a7c8d224bea52 is 50, key is test_row_0/A:col10/1733235363907/Put/seqid=0 2024-12-03T14:16:05,386 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:05,386 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d31a2807683a40d2acb753b1002670da is 50, key is test_row_0/B:col10/1733235363907/Put/seqid=0 2024-12-03T14:16:05,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742333_1509 (size=13119) 2024-12-03T14:16:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742334_1510 (size=13119) 2024-12-03T14:16:05,458 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:05,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5899c7af65574162a8f25dce89c9d6ba is 50, key is test_row_0/A:col10/1733235365698/Put/seqid=0 2024-12-03T14:16:05,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742335_1511 (size=17181) 2024-12-03T14:16:05,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5899c7af65574162a8f25dce89c9d6ba 2024-12-03T14:16:05,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8e78ac0ce3d246a0a89aa1ace4d3432e is 50, key is test_row_0/B:col10/1733235365698/Put/seqid=0 2024-12-03T14:16:05,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742336_1512 (size=12301) 2024-12-03T14:16:05,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8e78ac0ce3d246a0a89aa1ace4d3432e 2024-12-03T14:16:05,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df02fa8ddd714a05856a7489bb91fcb6 is 50, key is test_row_0/C:col10/1733235365698/Put/seqid=0 2024-12-03T14:16:05,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742337_1513 (size=12301) 2024-12-03T14:16:05,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df02fa8ddd714a05856a7489bb91fcb6 2024-12-03T14:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5899c7af65574162a8f25dce89c9d6ba as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba 2024-12-03T14:16:05,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba, entries=250, sequenceid=379, filesize=16.8 K 2024-12-03T14:16:05,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8e78ac0ce3d246a0a89aa1ace4d3432e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e 2024-12-03T14:16:05,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e, entries=150, sequenceid=379, filesize=12.0 K 2024-12-03T14:16:05,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/df02fa8ddd714a05856a7489bb91fcb6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6 2024-12-03T14:16:05,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6, entries=150, sequenceid=379, filesize=12.0 K 2024-12-03T14:16:05,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=67.09 KB/68700 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 43ms, sequenceid=379, compaction requested=true 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T14:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:05,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:05,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:05,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/beec986f334546dc8d63941898078624 is 50, key is test_row_0/A:col10/1733235365739/Put/seqid=0 2024-12-03T14:16:05,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742338_1514 (size=17181) 2024-12-03T14:16:05,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235425782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,791 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/92dea45e414e437db85a7c8d224bea52 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/92dea45e414e437db85a7c8d224bea52 2024-12-03T14:16:05,792 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d31a2807683a40d2acb753b1002670da as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d31a2807683a40d2acb753b1002670da 2024-12-03T14:16:05,795 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into d31a2807683a40d2acb753b1002670da(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:05,795 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 92dea45e414e437db85a7c8d224bea52(size=12.8 K), total size for store is 29.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:05,795 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235365378; duration=0sec 2024-12-03T14:16:05,795 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235365378; duration=0sec 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:05,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:05,796 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:05,796 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8c3cd94205b411f9d3866aa0b4688e0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=48.8 K 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c3cd94205b411f9d3866aa0b4688e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733235361371 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting fa90006efb094fcea194ee13aa05aee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235361719 2024-12-03T14:16:05,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235425783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0938c2b546b54021b0f7dbb887df7769, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235363907 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235425785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,797 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting df02fa8ddd714a05856a7489bb91fcb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733235364561 2024-12-03T14:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,803 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:05,803 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/530d1cd238e946af9604bc7efb4928f7 is 50, key is test_row_0/C:col10/1733235365698/Put/seqid=0 2024-12-03T14:16:05,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742339_1515 (size=13153) 2024-12-03T14:16:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T14:16:05,866 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:05,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-03T14:16:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:05,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:05,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:05,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235425895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235425906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235425906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:06,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235426116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235426126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235426126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/beec986f334546dc8d63941898078624 2024-12-03T14:16:06,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d86d0b37dbc144b1aad05aa4ca1c07d9 is 50, key is test_row_0/B:col10/1733235365739/Put/seqid=0 2024-12-03T14:16:06,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742340_1516 (size=12301) 2024-12-03T14:16:06,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d86d0b37dbc144b1aad05aa4ca1c07d9 2024-12-03T14:16:06,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/25225397d86d4023bd361bfe6a3dba19 is 50, key is test_row_0/C:col10/1733235365739/Put/seqid=0 2024-12-03T14:16:06,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742341_1517 (size=12301) 2024-12-03T14:16:06,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:06,214 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/530d1cd238e946af9604bc7efb4928f7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/530d1cd238e946af9604bc7efb4928f7 2024-12-03T14:16:06,217 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into 530d1cd238e946af9604bc7efb4928f7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:06,218 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:06,218 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=12, startTime=1733235365742; duration=0sec 2024-12-03T14:16:06,218 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:06,218 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:06,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235426426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235426436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235426436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:06,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/25225397d86d4023bd361bfe6a3dba19 2024-12-03T14:16:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/beec986f334546dc8d63941898078624 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624 2024-12-03T14:16:06,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624, entries=250, sequenceid=394, filesize=16.8 K 2024-12-03T14:16:06,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/d86d0b37dbc144b1aad05aa4ca1c07d9 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9 2024-12-03T14:16:06,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T14:16:06,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/25225397d86d4023bd361bfe6a3dba19 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19 2024-12-03T14:16:06,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T14:16:06,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 843ms, sequenceid=394, compaction requested=true 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:06,585 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:06,585 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:16:06,586 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,586 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,586 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d31a2807683a40d2acb753b1002670da, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.8 K 2024-12-03T14:16:06,586 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/92dea45e414e437db85a7c8d224bea52, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=46.4 K 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d31a2807683a40d2acb753b1002670da, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235363907 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92dea45e414e437db85a7c8d224bea52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1733235363907 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e78ac0ce3d246a0a89aa1ace4d3432e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733235364561 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5899c7af65574162a8f25dce89c9d6ba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733235364551 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d86d0b37dbc144b1aad05aa4ca1c07d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235365725 2024-12-03T14:16:06,586 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting beec986f334546dc8d63941898078624, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235365725 2024-12-03T14:16:06,592 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:06,593 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/a3a070b07de942c18f22ab8ff76744e3 is 50, key is test_row_0/A:col10/1733235365739/Put/seqid=0 2024-12-03T14:16:06,593 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:06,593 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/342f8d37525746cb86736a7c5b9dd1bd is 50, key is test_row_0/B:col10/1733235365739/Put/seqid=0 2024-12-03T14:16:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742342_1518 (size=13221) 2024-12-03T14:16:06,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742343_1519 (size=13221) 2024-12-03T14:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:06,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:16:06,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:06,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:06,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:06,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:06,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:06,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:06,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/12c3f0980a45440ea4adafaa0854295d is 50, key is test_row_0/A:col10/1733235366618/Put/seqid=0 2024-12-03T14:16:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742344_1520 (size=14741) 2024-12-03T14:16:06,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235426676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235426684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235426786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235426796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,910 INFO [Thread-1973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1973. 2024-12-03T14:16:06,910 DEBUG [Thread-1973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:06,910 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,910 DEBUG [Thread-1973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:06,912 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-12-03T14:16:06,912 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:06,912 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,912 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:06,912 INFO [Thread-1971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1971. 2024-12-03T14:16:06,912 DEBUG [Thread-1971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:06,912 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,912 DEBUG [Thread-1971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:06,916 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-12-03T14:16:06,916 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:06,916 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,916 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:06,917 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-12-03T14:16:06,917 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:06,917 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,918 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:06,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:06,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53728 deadline: 1733235426936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:06,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53708 deadline: 1733235426946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53720 deadline: 1733235426946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:06,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235426996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:07,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235427006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,007 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/a3a070b07de942c18f22ab8ff76744e3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a3a070b07de942c18f22ab8ff76744e3 2024-12-03T14:16:07,011 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into a3a070b07de942c18f22ab8ff76744e3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:07,011 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:07,011 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235366585; duration=0sec 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:07,012 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:07,014 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/342f8d37525746cb86736a7c5b9dd1bd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/342f8d37525746cb86736a7c5b9dd1bd 2024-12-03T14:16:07,017 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into 342f8d37525746cb86736a7c5b9dd1bd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:07,017 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:07,017 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235366585; duration=0sec 2024-12-03T14:16:07,018 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:07,018 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:07,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/12c3f0980a45440ea4adafaa0854295d 2024-12-03T14:16:07,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8c8704aafe7e4b8d947f3da5a1d51a88 is 50, key is test_row_0/B:col10/1733235366618/Put/seqid=0 2024-12-03T14:16:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742345_1521 (size=12301) 2024-12-03T14:16:07,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235427306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235427316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8c8704aafe7e4b8d947f3da5a1d51a88 2024-12-03T14:16:07,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/323cca0286044ec5b253830d381627c6 is 50, key is test_row_0/C:col10/1733235366618/Put/seqid=0 2024-12-03T14:16:07,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742346_1522 (size=12301) 2024-12-03T14:16:07,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53696 deadline: 1733235427817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53716 deadline: 1733235427826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 , the old value is region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=2 because the exception is null or not the one we care about 2024-12-03T14:16:07,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/323cca0286044ec5b253830d381627c6 2024-12-03T14:16:07,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:07,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:07,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:07,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/12c3f0980a45440ea4adafaa0854295d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d 2024-12-03T14:16:07,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d, entries=200, sequenceid=415, filesize=14.4 K 2024-12-03T14:16:07,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/8c8704aafe7e4b8d947f3da5a1d51a88 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88 2024-12-03T14:16:07,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88, entries=150, sequenceid=415, filesize=12.0 K 2024-12-03T14:16:07,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/323cca0286044ec5b253830d381627c6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6 2024-12-03T14:16:07,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6, entries=150, sequenceid=415, filesize=12.0 K 2024-12-03T14:16:07,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1262ms, sequenceid=415, compaction requested=true 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:07,881 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:07,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:07,881 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:07,882 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:07,883 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:07,883 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/C is initiating minor compaction (all files) 2024-12-03T14:16:07,883 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/C in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:07,883 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/530d1cd238e946af9604bc7efb4928f7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.9 K 2024-12-03T14:16:07,883 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 530d1cd238e946af9604bc7efb4928f7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733235364561 2024-12-03T14:16:07,884 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 25225397d86d4023bd361bfe6a3dba19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235365725 2024-12-03T14:16:07,884 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 323cca0286044ec5b253830d381627c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733235365781 2024-12-03T14:16:07,890 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#C#compaction#446 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:07,890 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/c0def9618b1d47b1a3ec1ae6443c212d is 50, key is test_row_0/C:col10/1733235366618/Put/seqid=0 2024-12-03T14:16:07,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742347_1523 (size=13255) 2024-12-03T14:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:07,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:16:07,948 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:07,948 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:07,948 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:07,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:07,948 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:07,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5d6cf14cc77a460583d3fe774a09760a is 50, key is test_row_0/A:col10/1733235366683/Put/seqid=0 2024-12-03T14:16:07,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742348_1524 (size=12301) 2024-12-03T14:16:07,958 INFO [Thread-1960 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1960. 2024-12-03T14:16:07,958 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-12-03T14:16:07,958 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:07,958 DEBUG [Thread-1960 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:07,958 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,958 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,958 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,958 DEBUG [Thread-1960 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:07,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:07,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:08,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:08,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,303 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/c0def9618b1d47b1a3ec1ae6443c212d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/c0def9618b1d47b1a3ec1ae6443c212d 2024-12-03T14:16:08,310 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/C of e0c0e6a2f069c0cb795f0cc0acfdf529 into c0def9618b1d47b1a3ec1ae6443c212d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:08,311 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:08,311 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/C, priority=13, startTime=1733235367881; duration=0sec 2024-12-03T14:16:08,311 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:08,311 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:08,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5d6cf14cc77a460583d3fe774a09760a 2024-12-03T14:16:08,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/fdf103eb2071444ba519feeccc05a60f is 50, key is test_row_0/B:col10/1733235366683/Put/seqid=0 2024-12-03T14:16:08,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742349_1525 (size=12301) 2024-12-03T14:16:08,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/fdf103eb2071444ba519feeccc05a60f 2024-12-03T14:16:08,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4ea8fe3d43a342e1861fc5449a0474dd is 50, key is test_row_0/C:col10/1733235366683/Put/seqid=0 2024-12-03T14:16:08,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742350_1526 (size=12301) 2024-12-03T14:16:08,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,839 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-12-03T14:16:08,839 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-12-03T14:16:08,839 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:08,839 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:08,839 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:08,839 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:08,839 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:08,839 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:08,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:08,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:08,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:08,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:09,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. as already flushing 2024-12-03T14:16:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:09,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:09,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4ea8fe3d43a342e1861fc5449a0474dd 2024-12-03T14:16:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/5d6cf14cc77a460583d3fe774a09760a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a 2024-12-03T14:16:09,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a, entries=150, sequenceid=432, filesize=12.0 K 2024-12-03T14:16:09,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/fdf103eb2071444ba519feeccc05a60f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f 2024-12-03T14:16:09,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f, entries=150, sequenceid=432, filesize=12.0 K 2024-12-03T14:16:09,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/4ea8fe3d43a342e1861fc5449a0474dd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4ea8fe3d43a342e1861fc5449a0474dd 2024-12-03T14:16:09,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4ea8fe3d43a342e1861fc5449a0474dd, entries=150, sequenceid=432, filesize=12.0 K 2024-12-03T14:16:09,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1254ms, sequenceid=432, compaction requested=true 2024-12-03T14:16:09,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c0e6a2f069c0cb795f0cc0acfdf529:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:09,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/B is initiating minor compaction (all files) 2024-12-03T14:16:09,203 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): e0c0e6a2f069c0cb795f0cc0acfdf529/A is initiating minor compaction (all files) 2024-12-03T14:16:09,203 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/B in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:09,204 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c0e6a2f069c0cb795f0cc0acfdf529/A in TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:09,204 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/342f8d37525746cb86736a7c5b9dd1bd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=36.9 K 2024-12-03T14:16:09,204 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a3a070b07de942c18f22ab8ff76744e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp, totalSize=39.3 K 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 342f8d37525746cb86736a7c5b9dd1bd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235365725 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a070b07de942c18f22ab8ff76744e3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733235365725 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8704aafe7e4b8d947f3da5a1d51a88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733235365781 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c3f0980a45440ea4adafaa0854295d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733235365781 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf103eb2071444ba519feeccc05a60f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733235366673 2024-12-03T14:16:09,204 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d6cf14cc77a460583d3fe774a09760a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733235366673 2024-12-03T14:16:09,214 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:09,214 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c0e6a2f069c0cb795f0cc0acfdf529#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:09,215 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/004202d81ea145e8b32d95b310b40ce0 is 50, key is test_row_0/B:col10/1733235366683/Put/seqid=0 2024-12-03T14:16:09,215 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/63df1832e18343cea68a0a64f95cb239 is 50, key is test_row_0/A:col10/1733235366683/Put/seqid=0 2024-12-03T14:16:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742352_1528 (size=13323) 2024-12-03T14:16:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742351_1527 (size=13323) 2024-12-03T14:16:09,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:09,258 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e0c0e6a2f069c0cb795f0cc0acfdf529 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=A 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=B 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c0e6a2f069c0cb795f0cc0acfdf529, store=C 2024-12-03T14:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:09,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/f8681935b8ae410ea2b44a7eea6dee7a is 50, key is test_row_0/A:col10/1733235367956/Put/seqid=0 2024-12-03T14:16:09,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742353_1529 (size=12301) 2024-12-03T14:16:09,627 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/63df1832e18343cea68a0a64f95cb239 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/63df1832e18343cea68a0a64f95cb239 2024-12-03T14:16:09,627 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/004202d81ea145e8b32d95b310b40ce0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/004202d81ea145e8b32d95b310b40ce0 2024-12-03T14:16:09,631 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/A of e0c0e6a2f069c0cb795f0cc0acfdf529 into 63df1832e18343cea68a0a64f95cb239(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:09,631 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c0e6a2f069c0cb795f0cc0acfdf529/B of e0c0e6a2f069c0cb795f0cc0acfdf529 into 004202d81ea145e8b32d95b310b40ce0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:09,631 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/B, priority=13, startTime=1733235369203; duration=0sec 2024-12-03T14:16:09,631 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529., storeName=e0c0e6a2f069c0cb795f0cc0acfdf529/A, priority=13, startTime=1733235369202; duration=0sec 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:A 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:B 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T14:16:09,631 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. because compaction request was cancelled 2024-12-03T14:16:09,632 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c0e6a2f069c0cb795f0cc0acfdf529:C 2024-12-03T14:16:09,665 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/f8681935b8ae410ea2b44a7eea6dee7a 2024-12-03T14:16:09,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/196059b0fa6445898efb9deb80d3f0fa is 50, key is test_row_0/B:col10/1733235367956/Put/seqid=0 2024-12-03T14:16:09,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742354_1530 (size=12301) 2024-12-03T14:16:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:10,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/196059b0fa6445898efb9deb80d3f0fa 2024-12-03T14:16:10,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/80062d783c2241cdb2447d16aaf4e617 is 50, key is test_row_0/C:col10/1733235367956/Put/seqid=0 2024-12-03T14:16:10,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742355_1531 (size=12301) 2024-12-03T14:16:10,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/80062d783c2241cdb2447d16aaf4e617 2024-12-03T14:16:10,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/A/f8681935b8ae410ea2b44a7eea6dee7a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/f8681935b8ae410ea2b44a7eea6dee7a 2024-12-03T14:16:10,500 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/f8681935b8ae410ea2b44a7eea6dee7a, entries=150, sequenceid=440, filesize=12.0 K 2024-12-03T14:16:10,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/B/196059b0fa6445898efb9deb80d3f0fa as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/196059b0fa6445898efb9deb80d3f0fa 2024-12-03T14:16:10,505 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/196059b0fa6445898efb9deb80d3f0fa, entries=150, sequenceid=440, filesize=12.0 K 2024-12-03T14:16:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/.tmp/C/80062d783c2241cdb2447d16aaf4e617 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/80062d783c2241cdb2447d16aaf4e617 2024-12-03T14:16:10,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/80062d783c2241cdb2447d16aaf4e617, entries=150, sequenceid=440, filesize=12.0 K 2024-12-03T14:16:10,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e0c0e6a2f069c0cb795f0cc0acfdf529 in 1252ms, sequenceid=440, compaction requested=true 2024-12-03T14:16:10,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e0c0e6a2f069c0cb795f0cc0acfdf529: 2024-12-03T14:16:10,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:10,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-03T14:16:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-03T14:16:10,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T14:16:10,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.6410 sec 2024-12-03T14:16:10,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 4.6460 sec 2024-12-03T14:16:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T14:16:14,047 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1702 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5105 rows 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1698 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5093 rows 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1695 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5078 rows 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1691 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5073 rows 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1708 2024-12-03T14:16:14,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5120 rows 2024-12-03T14:16:14,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:16:14,048 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:16:14,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:14,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:14,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:14,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:16:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T14:16:14,056 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235374055"}]},"ts":"1733235374055"} 2024-12-03T14:16:14,058 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:16:14,058 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:16:14,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:16:14,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, UNASSIGN}] 2024-12-03T14:16:14,061 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, UNASSIGN 2024-12-03T14:16:14,062 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e0c0e6a2f069c0cb795f0cc0acfdf529, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:14,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, UNASSIGN because future has completed 2024-12-03T14:16:14,063 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:16:14,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:16:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T14:16:14,216 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:14,216 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:16:14,216 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing e0c0e6a2f069c0cb795f0cc0acfdf529, disabling compactions & flushes 2024-12-03T14:16:14,216 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:14,216 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:14,216 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. after waiting 0 ms 2024-12-03T14:16:14,216 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:14,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/8e1497654d4847d3bffac1db90d2098c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/08e6d4f9feb14d4185ac2e7b7a04bb63, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/6c2f72bc26b44b81a8d9bdce620133aa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7f7eeef3bd8f4417b2f8863135c1955d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/53521c7542034bb68984bb48695d4103, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/35afb14f6dc044eeb64cddd8430a2792, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/92dea45e414e437db85a7c8d224bea52, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a3a070b07de942c18f22ab8ff76744e3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a] to archive 2024-12-03T14:16:14,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:14,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c23b9083da20456c9d06c1a3288f0766 2024-12-03T14:16:14,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/94cfd7885b434dbdbb6c8924516d4bed 2024-12-03T14:16:14,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/811fa6344977452a92dfd98b105e2a5a 2024-12-03T14:16:14,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/b131f8aa067a4fd7bb98e2cf7b9b34dc 2024-12-03T14:16:14,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/bd742f2ad0524acf96980f88a7922459 2024-12-03T14:16:14,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/8e1497654d4847d3bffac1db90d2098c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/8e1497654d4847d3bffac1db90d2098c 2024-12-03T14:16:14,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3a40ebd7b53a47d19b7be533cc92fb3f 2024-12-03T14:16:14,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/357a197aba004ea6a7064eed1b090d16 2024-12-03T14:16:14,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/08e6d4f9feb14d4185ac2e7b7a04bb63 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/08e6d4f9feb14d4185ac2e7b7a04bb63 2024-12-03T14:16:14,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc05df2727194c33bd8c99587e81038e 2024-12-03T14:16:14,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/6c2f72bc26b44b81a8d9bdce620133aa to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/6c2f72bc26b44b81a8d9bdce620133aa 2024-12-03T14:16:14,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7c7ba999193e4d8592485798467a1e5c 2024-12-03T14:16:14,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/1ff9335af3c643ffb7030935c0a66467 2024-12-03T14:16:14,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/3cd495d9cd6b4ba58d85272f68de0770 2024-12-03T14:16:14,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7f7eeef3bd8f4417b2f8863135c1955d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/7f7eeef3bd8f4417b2f8863135c1955d 2024-12-03T14:16:14,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/ddc4966ec38347a4bc04b33bb5fa93f8 2024-12-03T14:16:14,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/cab0b6e5a4914d0892bb84b083ffcda5 2024-12-03T14:16:14,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/53521c7542034bb68984bb48695d4103 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/53521c7542034bb68984bb48695d4103 2024-12-03T14:16:14,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a812d4551ee846e09a9170d0ccd5c79c 2024-12-03T14:16:14,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/37b81440e4d24ecda239843d9d3e62f7 2024-12-03T14:16:14,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/097d895393e44633b467f8bc19c9319f 2024-12-03T14:16:14,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/35afb14f6dc044eeb64cddd8430a2792 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/35afb14f6dc044eeb64cddd8430a2792 2024-12-03T14:16:14,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/fc8031cd8b79426d99d4bd3380a4af31 2024-12-03T14:16:14,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/c01dd381a3ec4aef80af32a8f550e290 2024-12-03T14:16:14,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/92dea45e414e437db85a7c8d224bea52 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/92dea45e414e437db85a7c8d224bea52 2024-12-03T14:16:14,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5899c7af65574162a8f25dce89c9d6ba 2024-12-03T14:16:14,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/beec986f334546dc8d63941898078624 2024-12-03T14:16:14,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a3a070b07de942c18f22ab8ff76744e3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/a3a070b07de942c18f22ab8ff76744e3 2024-12-03T14:16:14,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/12c3f0980a45440ea4adafaa0854295d 2024-12-03T14:16:14,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/5d6cf14cc77a460583d3fe774a09760a 2024-12-03T14:16:14,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/b7e23c5a91bb4123be35032d57982ca4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f7656ca8959247a6b332ac451a86357e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e66a172f600e4a1ebed4e3ee9a5d8e52, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/37ea5247ac0a4df5bbba878892818355, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/55d154e1c51441ad914afa8d28eabb15, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f36a68b4fdd84418a4a3b9e10a563de6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d31a2807683a40d2acb753b1002670da, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/342f8d37525746cb86736a7c5b9dd1bd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f] to archive 2024-12-03T14:16:14,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:14,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6c6a1b824bd42528ae91e47e38cb8de 2024-12-03T14:16:14,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/a1ed96797b784b3d820d5891a23c9686 2024-12-03T14:16:14,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/371cbd2836734b9d83d7fc93174cb3f1 2024-12-03T14:16:14,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/753e84cba117493ba0050ccddfa78bc7 2024-12-03T14:16:14,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/b7e23c5a91bb4123be35032d57982ca4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/b7e23c5a91bb4123be35032d57982ca4 2024-12-03T14:16:14,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ab37980f4d98426ea92eee94363dee3e 2024-12-03T14:16:14,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/71197e87b76340d7a19229aabf4ef539 2024-12-03T14:16:14,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f7656ca8959247a6b332ac451a86357e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f7656ca8959247a6b332ac451a86357e 2024-12-03T14:16:14,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/db344e1288e44a18bc3ff46f9460f107 2024-12-03T14:16:14,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e6eecb14595d43f2810d43f329ec812e 2024-12-03T14:16:14,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e66a172f600e4a1ebed4e3ee9a5d8e52 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/e66a172f600e4a1ebed4e3ee9a5d8e52 2024-12-03T14:16:14,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/7c82d1704bf54e1a9a390f3f50904f66 2024-12-03T14:16:14,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/74ccbbbea1224b2783b17d310cb23bd8 2024-12-03T14:16:14,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/37ea5247ac0a4df5bbba878892818355 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/37ea5247ac0a4df5bbba878892818355 2024-12-03T14:16:14,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2bd58e4167194ec78863a2b5fbb3462e 2024-12-03T14:16:14,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2ba1d629ec584e5393656711147f72e1 2024-12-03T14:16:14,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/55d154e1c51441ad914afa8d28eabb15 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/55d154e1c51441ad914afa8d28eabb15 2024-12-03T14:16:14,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d76eafe9a7a24dcb9909372ccf4f9e82 2024-12-03T14:16:14,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8f63b3ad4d554079aec970eeceb628a3 2024-12-03T14:16:14,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/ed9c9693a0d2450db5e4b4b893f7409e 2024-12-03T14:16:14,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f36a68b4fdd84418a4a3b9e10a563de6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f36a68b4fdd84418a4a3b9e10a563de6 2024-12-03T14:16:14,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/f049c458feae4e33a4b1dfc70564a4ff 2024-12-03T14:16:14,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/86bad225264a47158fe2bb2af346e293 2024-12-03T14:16:14,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d31a2807683a40d2acb753b1002670da to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d31a2807683a40d2acb753b1002670da 2024-12-03T14:16:14,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/2e341302e91d4fcd8fd586369b190e72 2024-12-03T14:16:14,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8e78ac0ce3d246a0a89aa1ace4d3432e 2024-12-03T14:16:14,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/342f8d37525746cb86736a7c5b9dd1bd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/342f8d37525746cb86736a7c5b9dd1bd 2024-12-03T14:16:14,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/d86d0b37dbc144b1aad05aa4ca1c07d9 2024-12-03T14:16:14,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/8c8704aafe7e4b8d947f3da5a1d51a88 2024-12-03T14:16:14,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/fdf103eb2071444ba519feeccc05a60f 2024-12-03T14:16:14,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/87a8d74b44404f68a075c999946fe971, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8bd1e5ffb814741b54ea524974e44db, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fadd25f0102e433395a1249cbcc78bc6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/6fdad1107a21445b88048acf7173eace, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b55bba747b4a4f6088bb6b83aca22d84, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8c3cd94205b411f9d3866aa0b4688e0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/530d1cd238e946af9604bc7efb4928f7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6] to archive 2024-12-03T14:16:14,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:14,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/355116a291b94c1e9e9420680fe698bd 2024-12-03T14:16:14,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/2bae927f39ba4858b7e85182230cad4b 2024-12-03T14:16:14,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b52900265a7e4cb3a2f4913ee2427f4c 2024-12-03T14:16:14,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/1f51aa93f43b4f68b4bccf0d0376f37a 2024-12-03T14:16:14,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/87a8d74b44404f68a075c999946fe971 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/87a8d74b44404f68a075c999946fe971 2024-12-03T14:16:14,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/e61d2e00f9784e84bd2c2536ac6da291 2024-12-03T14:16:14,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d36ef5944ca84663bb4e55215a884734 2024-12-03T14:16:14,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8bd1e5ffb814741b54ea524974e44db to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8bd1e5ffb814741b54ea524974e44db 2024-12-03T14:16:14,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4adcbebf0682470b8058ccb68f62346a 2024-12-03T14:16:14,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/42bcc2f7694b40cf8fb6a89a387d899a 2024-12-03T14:16:14,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fadd25f0102e433395a1249cbcc78bc6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fadd25f0102e433395a1249cbcc78bc6 2024-12-03T14:16:14,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f5c4f11ac03d49a0ac35b1c254600b00 2024-12-03T14:16:14,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/d05e13c5c3fe44ffb40cd0dad5a4a0c8 2024-12-03T14:16:14,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/6fdad1107a21445b88048acf7173eace to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/6fdad1107a21445b88048acf7173eace 2024-12-03T14:16:14,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4a49d63d2c26454d9140a35e76fdf7c0 2024-12-03T14:16:14,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df13160206114db3b0b58803379be247 2024-12-03T14:16:14,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b55bba747b4a4f6088bb6b83aca22d84 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/b55bba747b4a4f6088bb6b83aca22d84 2024-12-03T14:16:14,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fe224c8c42cc4a8cb86ba27390c37815 2024-12-03T14:16:14,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/46bb13408465487caf1ce81cc1c23ec5 2024-12-03T14:16:14,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9bdf23c91c59401bb6d76906e3f1fcc7 2024-12-03T14:16:14,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8c3cd94205b411f9d3866aa0b4688e0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/f8c3cd94205b411f9d3866aa0b4688e0 2024-12-03T14:16:14,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/9f87147a052144e6927e546cf1263387 2024-12-03T14:16:14,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/fa90006efb094fcea194ee13aa05aee6 2024-12-03T14:16:14,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/0938c2b546b54021b0f7dbb887df7769 2024-12-03T14:16:14,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/530d1cd238e946af9604bc7efb4928f7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/530d1cd238e946af9604bc7efb4928f7 2024-12-03T14:16:14,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/df02fa8ddd714a05856a7489bb91fcb6 2024-12-03T14:16:14,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/25225397d86d4023bd361bfe6a3dba19 2024-12-03T14:16:14,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/323cca0286044ec5b253830d381627c6 2024-12-03T14:16:14,315 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-12-03T14:16:14,316 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529. 2024-12-03T14:16:14,316 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for e0c0e6a2f069c0cb795f0cc0acfdf529: Waiting for close lock at 1733235374216Running coprocessor pre-close hooks at 1733235374216Disabling compacts and flushes for region at 1733235374216Disabling writes for close at 1733235374216Writing region close event to WAL at 1733235374313 (+97 ms)Running coprocessor post-close hooks at 1733235374316 (+3 ms)Closed at 1733235374316 2024-12-03T14:16:14,317 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:14,318 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e0c0e6a2f069c0cb795f0cc0acfdf529, regionState=CLOSED 2024-12-03T14:16:14,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:16:14,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T14:16:14,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure e0c0e6a2f069c0cb795f0cc0acfdf529, server=3920e6a7c3b5,42177,1733235217010 in 257 msec 2024-12-03T14:16:14,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-03T14:16:14,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c0e6a2f069c0cb795f0cc0acfdf529, UNASSIGN in 262 msec 2024-12-03T14:16:14,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T14:16:14,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 265 msec 2024-12-03T14:16:14,327 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235374327"}]},"ts":"1733235374327"} 2024-12-03T14:16:14,328 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:16:14,328 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:16:14,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 279 msec 2024-12-03T14:16:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T14:16:14,377 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:14,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:16:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,382 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T14:16:14,384 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,388 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:14,393 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/recovered.edits] 2024-12-03T14:16:14,397 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/63df1832e18343cea68a0a64f95cb239 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/63df1832e18343cea68a0a64f95cb239 2024-12-03T14:16:14,399 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/f8681935b8ae410ea2b44a7eea6dee7a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/A/f8681935b8ae410ea2b44a7eea6dee7a 2024-12-03T14:16:14,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/004202d81ea145e8b32d95b310b40ce0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/004202d81ea145e8b32d95b310b40ce0 2024-12-03T14:16:14,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/196059b0fa6445898efb9deb80d3f0fa to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/B/196059b0fa6445898efb9deb80d3f0fa 2024-12-03T14:16:14,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4ea8fe3d43a342e1861fc5449a0474dd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/4ea8fe3d43a342e1861fc5449a0474dd 2024-12-03T14:16:14,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/80062d783c2241cdb2447d16aaf4e617 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/80062d783c2241cdb2447d16aaf4e617 2024-12-03T14:16:14,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/c0def9618b1d47b1a3ec1ae6443c212d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/C/c0def9618b1d47b1a3ec1ae6443c212d 2024-12-03T14:16:14,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/recovered.edits/445.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529/recovered.edits/445.seqid 2024-12-03T14:16:14,413 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/e0c0e6a2f069c0cb795f0cc0acfdf529 2024-12-03T14:16:14,413 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:16:14,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,419 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:16:14,421 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:16:14,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:16:14,423 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235374423"}]},"ts":"9223372036854775807"} 2024-12-03T14:16:14,426 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:16:14,426 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e0c0e6a2f069c0cb795f0cc0acfdf529, NAME => 'TestAcidGuarantees,,1733235344710.e0c0e6a2f069c0cb795f0cc0acfdf529.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:16:14,426 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:16:14,426 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235374426"}]},"ts":"9223372036854775807"} 2024-12-03T14:16:14,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:16:14,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-03T14:16:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T14:16:14,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:16:14,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:14,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252 (was 248) - Thread LEAK? -, OpenFileDescriptor=465 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 351) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10027 (was 10068) 2024-12-03T14:16:14,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=10026 2024-12-03T14:16:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:16:14,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:16:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:14,516 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:16:14,516 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:14,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-03T14:16:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:14,517 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:16:14,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742356_1532 (size=1061) 2024-12-03T14:16:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:14,928 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f 2024-12-03T14:16:14,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742357_1533 (size=53) 2024-12-03T14:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c2b13e21c278fc9a8535a3ecbf922878, disabling compactions & flushes 2024-12-03T14:16:15,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. after waiting 0 ms 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,340 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,340 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2b13e21c278fc9a8535a3ecbf922878: Waiting for close lock at 1733235375340Disabling compacts and flushes for region at 1733235375340Disabling writes for close at 1733235375340Writing region close event to WAL at 1733235375340Closed at 1733235375340 2024-12-03T14:16:15,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:16:15,344 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733235375343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733235375343"}]},"ts":"1733235375343"} 2024-12-03T14:16:15,348 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:16:15,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:16:15,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235375350"}]},"ts":"1733235375350"} 2024-12-03T14:16:15,353 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T14:16:15,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, ASSIGN}] 2024-12-03T14:16:15,354 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, ASSIGN 2024-12-03T14:16:15,354 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, ASSIGN; state=OFFLINE, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=false 2024-12-03T14:16:15,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:15,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, ASSIGN because future has completed 2024-12-03T14:16:15,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:16:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:15,665 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,666 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:16:15,666 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,666 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:16:15,667 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,667 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,669 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,671 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:15,671 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName A 2024-12-03T14:16:15,671 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:15,672 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:15,672 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,674 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:15,674 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName B 2024-12-03T14:16:15,674 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:15,675 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:15,675 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,676 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:15,676 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName C 2024-12-03T14:16:15,676 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:15,677 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:15,677 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,677 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,678 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,679 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,680 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,680 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,681 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:16:15,682 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,684 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:16:15,684 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened c2b13e21c278fc9a8535a3ecbf922878; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73775677, jitterRate=0.09934325516223907}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:16:15,684 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:15,685 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for c2b13e21c278fc9a8535a3ecbf922878: Running coprocessor pre-open hook at 1733235375667Writing region info on filesystem at 1733235375667Initializing all the Stores at 1733235375668 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235375668Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235375669 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235375669Cleaning up temporary data from old regions at 1733235375680 (+11 ms)Running coprocessor post-open hooks at 1733235375684 (+4 ms)Region opened successfully at 1733235375685 (+1 ms) 2024-12-03T14:16:15,686 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., pid=144, masterSystemTime=1733235375659 2024-12-03T14:16:15,688 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,688 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:15,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=OPEN, openSeqNum=2, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:15,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:16:15,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T14:16:15,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 in 184 msec 2024-12-03T14:16:15,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-03T14:16:15,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, ASSIGN in 340 msec 2024-12-03T14:16:15,696 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:16:15,696 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235375696"}]},"ts":"1733235375696"} 2024-12-03T14:16:15,697 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T14:16:15,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:16:15,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-12-03T14:16:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T14:16:16,667 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:16,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:16,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:16,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:16,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544a01ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:16,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c33011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:16,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:16,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:16,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:16:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3920e6a7c3b5,42837,1733235216263 2024-12-03T14:16:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41a57a72 2024-12-03T14:16:16,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:16:16,681 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:16:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T14:16:16,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:16:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742358_1534 (size=1097) 2024-12-03T14:16:17,102 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T14:16:17,102 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T14:16:17,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:16:17,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE}] 2024-12-03T14:16:17,112 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE 2024-12-03T14:16:17,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE because future has completed 2024-12-03T14:16:17,116 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:16:17,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:16:17,269 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,269 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:16:17,269 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing c2b13e21c278fc9a8535a3ecbf922878, disabling compactions & flushes 2024-12-03T14:16:17,269 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,269 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,269 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. after waiting 0 ms 2024-12-03T14:16:17,269 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,273 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T14:16:17,274 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,274 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for c2b13e21c278fc9a8535a3ecbf922878: Waiting for close lock at 1733235377269Running coprocessor pre-close hooks at 1733235377269Disabling compacts and flushes for region at 1733235377269Disabling writes for close at 1733235377269Writing region close event to WAL at 1733235377270 (+1 ms)Running coprocessor post-close hooks at 1733235377274 (+4 ms)Closed at 1733235377274 2024-12-03T14:16:17,274 WARN [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: c2b13e21c278fc9a8535a3ecbf922878 to self. 2024-12-03T14:16:17,277 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,278 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=CLOSED 2024-12-03T14:16:17,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:16:17,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-03T14:16:17,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 in 166 msec 2024-12-03T14:16:17,287 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE; state=CLOSED, location=3920e6a7c3b5,42177,1733235217010; forceNewPlan=false, retain=true 2024-12-03T14:16:17,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=OPENING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE because future has completed 2024-12-03T14:16:17,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:16:17,605 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,605 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:16:17,606 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,606 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:16:17,607 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,607 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,610 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,611 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:17,611 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName A 2024-12-03T14:16:17,613 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:17,613 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:17,613 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,614 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:17,614 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName B 2024-12-03T14:16:17,614 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:17,615 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:17,615 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,615 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T14:16:17,615 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2b13e21c278fc9a8535a3ecbf922878 columnFamilyName C 2024-12-03T14:16:17,615 DEBUG [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:17,616 INFO [StoreOpener-c2b13e21c278fc9a8535a3ecbf922878-1 {}] regionserver.HStore(327): Store=c2b13e21c278fc9a8535a3ecbf922878/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:16:17,616 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,616 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,616 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,617 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,618 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,618 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,618 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:16:17,619 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,620 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened c2b13e21c278fc9a8535a3ecbf922878; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62215924, jitterRate=-0.07291048765182495}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:16:17,620 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,621 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for c2b13e21c278fc9a8535a3ecbf922878: Running coprocessor pre-open hook at 1733235377607Writing region info on filesystem at 1733235377607Initializing all the Stores at 1733235377609 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235377609Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235377610 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733235377610Cleaning up temporary data from old regions at 1733235377618 (+8 ms)Running coprocessor post-open hooks at 1733235377620 (+2 ms)Region opened successfully at 1733235377621 (+1 ms) 2024-12-03T14:16:17,621 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., pid=149, masterSystemTime=1733235377598 2024-12-03T14:16:17,623 DEBUG [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,623 INFO [RS_OPEN_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,623 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=OPEN, openSeqNum=5, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:16:17,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-03T14:16:17,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 in 182 msec 2024-12-03T14:16:17,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T14:16:17,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, REOPEN/MOVE in 515 msec 2024-12-03T14:16:17,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T14:16:17,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-12-03T14:16:17,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-12-03T14:16:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T14:16:17,631 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:17,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f21127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3302f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd561b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@945f9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485389e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3988e09a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e5ddfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b561b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,645 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc7c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,647 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ebf96e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23cc2bde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264db749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,652 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d9562f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b888fc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,655 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abf9071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9ae62d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3617ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d72533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3920e6a7c3b5,42837,-1 for getting cluster id 2024-12-03T14:16:17,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:16:17,662 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd246842c-db5a-49e0-a354-4fa4c8943284' 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d246842c-db5a-49e0-a354-4fa4c8943284" 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468f1759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3920e6a7c3b5,42837,-1] 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:16:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:17,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:16:17,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ab011b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:16:17,664 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,664 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,664 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,664 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,665 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,665 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T14:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,666 DEBUG [Thread-2389 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,668 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,668 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:17,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:17,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,668 DEBUG [Thread-2391 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,668 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:16:17,668 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3920e6a7c3b5,42177,1733235217010, seqNum=-1] 2024-12-03T14:16:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:16:17,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:16:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5] 2024-12-03T14:16:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:17,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:17,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:17,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235437690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235437693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235377681, max timestamp=1733235377694Num uniques 859; 2024-12-03T14:16:17,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235377681, max timestamp=1733235377694Num uniques 859; 2024-12-03T14:16:17,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733235377681, max timestamp=1733235377694Num uniques 859; 2024-12-03T14:16:17,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:16:17,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:16:17,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120388ce8393fc094a32a7b8d7f9e5b4228e_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235377676/Put/seqid=0 2024-12-03T14:16:17,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742359_1535 (size=12154) 2024-12-03T14:16:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235437714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235437718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235437718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235437806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235437806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:17,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:17,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235437826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235437826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235437826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:17,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:17,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235438016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235438016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235438035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235438036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235438036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:18,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120388ce8393fc094a32a7b8d7f9e5b4228e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388ce8393fc094a32a7b8d7f9e5b4228e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:18,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/3af3a4814d424d9fa429b5dc36ed510f, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:18,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/3af3a4814d424d9fa429b5dc36ed510f is 175, key is test_row_0/A:col10/1733235377676/Put/seqid=0 2024-12-03T14:16:18,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742360_1536 (size=30955) 2024-12-03T14:16:18,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235438326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235438326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235438346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235438346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235438346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,531 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/3af3a4814d424d9fa429b5dc36ed510f 2024-12-03T14:16:18,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/01c9bec249a743ca9ee9b18befee9015 is 50, key is test_row_0/B:col10/1733235377676/Put/seqid=0 2024-12-03T14:16:18,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742361_1537 (size=12001) 2024-12-03T14:16:18,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/01c9bec249a743ca9ee9b18befee9015 2024-12-03T14:16:18,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f9d103383e044ab3bc11a9961e515b45 is 50, key is test_row_0/C:col10/1733235377676/Put/seqid=0 2024-12-03T14:16:18,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742362_1538 (size=12001) 2024-12-03T14:16:18,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235438839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235438839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235438856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235438856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235438856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:18,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:18,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:18,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f9d103383e044ab3bc11a9961e515b45 2024-12-03T14:16:19,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/3af3a4814d424d9fa429b5dc36ed510f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f 2024-12-03T14:16:19,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f, entries=150, sequenceid=15, filesize=30.2 K 2024-12-03T14:16:19,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/01c9bec249a743ca9ee9b18befee9015 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015 2024-12-03T14:16:19,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T14:16:19,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f9d103383e044ab3bc11a9961e515b45 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45 2024-12-03T14:16:19,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T14:16:19,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for c2b13e21c278fc9a8535a3ecbf922878 in 1333ms, sequenceid=15, compaction requested=false 2024-12-03T14:16:19,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:19,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T14:16:19,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:19,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-03T14:16:19,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:19,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:19,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b10a5a5822514fa4be9135e02a170c6e_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235377708/Put/seqid=0 2024-12-03T14:16:19,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742363_1539 (size=12154) 2024-12-03T14:16:19,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:19,455 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b10a5a5822514fa4be9135e02a170c6e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b10a5a5822514fa4be9135e02a170c6e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:19,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/60165803b62f4b39802323cf28460243, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:19,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/60165803b62f4b39802323cf28460243 is 175, key is test_row_0/A:col10/1733235377708/Put/seqid=0 2024-12-03T14:16:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742364_1540 (size=30955) 2024-12-03T14:16:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:19,832 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:16:19,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:19,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/60165803b62f4b39802323cf28460243 2024-12-03T14:16:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/5d71bd5d270f435a9452d5a0687d53d5 is 50, key is test_row_0/B:col10/1733235377708/Put/seqid=0 2024-12-03T14:16:19,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742365_1541 (size=12001) 2024-12-03T14:16:19,872 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/5d71bd5d270f435a9452d5a0687d53d5 2024-12-03T14:16:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235439869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235439869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235439870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235439870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235439876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9c7afb33fcb5467da91eac5d6abb1fb3 is 50, key is test_row_0/C:col10/1733235377708/Put/seqid=0 2024-12-03T14:16:19,878 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742366_1542 (size=12001) 2024-12-03T14:16:19,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235439976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235439976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235439977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235439977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235440186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235440186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235440186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235440186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9c7afb33fcb5467da91eac5d6abb1fb3 2024-12-03T14:16:20,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/60165803b62f4b39802323cf28460243 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243 2024-12-03T14:16:20,287 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243, entries=150, sequenceid=50, filesize=30.2 K 2024-12-03T14:16:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/5d71bd5d270f435a9452d5a0687d53d5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5 2024-12-03T14:16:20,291 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T14:16:20,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9c7afb33fcb5467da91eac5d6abb1fb3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3 2024-12-03T14:16:20,294 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T14:16:20,295 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for c2b13e21c278fc9a8535a3ecbf922878 in 1253ms, sequenceid=50, compaction requested=false 2024-12-03T14:16:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:20,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T14:16:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T14:16:20,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T14:16:20,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6270 sec 2024-12-03T14:16:20,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6330 sec 2024-12-03T14:16:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:20,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T14:16:20,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:20,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:20,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:20,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:20,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:20,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:20,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120363ef1a62ff9946139c4b43e92a0ffb8e_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742367_1543 (size=12154) 2024-12-03T14:16:20,506 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:20,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120363ef1a62ff9946139c4b43e92a0ffb8e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120363ef1a62ff9946139c4b43e92a0ffb8e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:20,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/df7ea8d7ae844b418b1f13dd276f80d6, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:20,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/df7ea8d7ae844b418b1f13dd276f80d6 is 175, key is test_row_0/A:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:20,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742368_1544 (size=30955) 2024-12-03T14:16:20,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235440514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235440517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235440518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235440518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235440626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235440626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235440626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235440626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235440836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235440836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235440836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235440836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:20,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/df7ea8d7ae844b418b1f13dd276f80d6 2024-12-03T14:16:20,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/22359731871e4095827caa8770a9ed89 is 50, key is test_row_0/B:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:20,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742369_1545 (size=12001) 2024-12-03T14:16:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235441145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235441146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235441146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235441146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/22359731871e4095827caa8770a9ed89 2024-12-03T14:16:21,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/75ab42fecebb4ee597bdc663fde95a2b is 50, key is test_row_0/C:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:21,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742370_1546 (size=12001) 2024-12-03T14:16:21,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/75ab42fecebb4ee597bdc663fde95a2b 2024-12-03T14:16:21,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/df7ea8d7ae844b418b1f13dd276f80d6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6 2024-12-03T14:16:21,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6, entries=150, sequenceid=63, filesize=30.2 K 2024-12-03T14:16:21,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/22359731871e4095827caa8770a9ed89 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89 2024-12-03T14:16:21,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89, entries=150, sequenceid=63, filesize=11.7 K 2024-12-03T14:16:21,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/75ab42fecebb4ee597bdc663fde95a2b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b 2024-12-03T14:16:21,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b, entries=150, sequenceid=63, filesize=11.7 K 2024-12-03T14:16:21,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c2b13e21c278fc9a8535a3ecbf922878 in 862ms, sequenceid=63, compaction requested=true 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:21,359 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:21,359 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:21,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:21,360 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:21,360 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:21,360 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:21,360 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:21,360 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:21,360 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:21,360 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=35.2 K 2024-12-03T14:16:21,360 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=90.7 K 2024-12-03T14:16:21,360 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:21,360 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6] 2024-12-03T14:16:21,361 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c9bec249a743ca9ee9b18befee9015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235377672 2024-12-03T14:16:21,361 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af3a4814d424d9fa429b5dc36ed510f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235377672 2024-12-03T14:16:21,361 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60165803b62f4b39802323cf28460243, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235377708 2024-12-03T14:16:21,361 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d71bd5d270f435a9452d5a0687d53d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235377708 2024-12-03T14:16:21,362 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 22359731871e4095827caa8770a9ed89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:21,362 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting df7ea8d7ae844b418b1f13dd276f80d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:21,373 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:21,373 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/09a0eaa0680d46a494708874787808aa is 50, key is test_row_0/B:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:21,373 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:21,385 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c0806df714fd4a9e8622f8c8791f1f8e_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:21,387 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c0806df714fd4a9e8622f8c8791f1f8e_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:21,387 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c0806df714fd4a9e8622f8c8791f1f8e_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742371_1547 (size=12104) 2024-12-03T14:16:21,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742372_1548 (size=4469) 2024-12-03T14:16:21,398 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/09a0eaa0680d46a494708874787808aa as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/09a0eaa0680d46a494708874787808aa 2024-12-03T14:16:21,402 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into 09a0eaa0680d46a494708874787808aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:21,403 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=13, startTime=1733235381359; duration=0sec 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:21,403 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:21,404 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:21,404 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=35.2 K 2024-12-03T14:16:21,404 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d103383e044ab3bc11a9961e515b45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733235377672 2024-12-03T14:16:21,404 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7afb33fcb5467da91eac5d6abb1fb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733235377708 2024-12-03T14:16:21,404 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ab42fecebb4ee597bdc663fde95a2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:21,410 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:21,411 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/23305b7358f1422597db91e79fadc86a is 50, key is test_row_0/C:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:21,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742373_1549 (size=12104) 2024-12-03T14:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:21,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235380512, max timestamp=1733235381658Num uniques 859; 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235380512, max timestamp=1733235381658Num uniques 859; 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235380512, max timestamp=1733235381658Num uniques 859; 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:21,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:21,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:21,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:21,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:21,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb1f85d8a6df4946a2adf6f4056af2e3_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235380517/Put/seqid=0 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235381670, max timestamp=1733235381681Num uniques 859; 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733235381670, max timestamp=1733235381681Num uniques 859; 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733235381670, max timestamp=1733235381681Num uniques 859; 2024-12-03T14:16:21,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T14:16:21,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742374_1550 (size=12154) 2024-12-03T14:16:21,684 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:21,687 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203cb1f85d8a6df4946a2adf6f4056af2e3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb1f85d8a6df4946a2adf6f4056af2e3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:21,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/e41ffeb3b3764468b191bea8512b5034, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:21,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/e41ffeb3b3764468b191bea8512b5034 is 175, key is test_row_0/A:col10/1733235380517/Put/seqid=0 2024-12-03T14:16:21,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742375_1551 (size=30955) 2024-12-03T14:16:21,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235441725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235441725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733235381718, max timestamp=1733235381727Num uniques 1118; 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733235381718, max timestamp=1733235381727Num uniques 1118; 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733235381718, max timestamp=1733235381727Num uniques 1118; 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T14:16:21,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235381747, max timestamp=1733235381765Num uniques 1027; 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733235381747, max timestamp=1733235381765Num uniques 1027; 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733235381747, max timestamp=1733235381765Num uniques 1027; 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-03T14:16:21,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-03T14:16:21,796 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#465 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:21,796 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/68001a821d174be48e96a1d2b82eab96 is 175, key is test_row_0/A:col10/1733235379868/Put/seqid=0 2024-12-03T14:16:21,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742376_1552 (size=31058) 2024-12-03T14:16:21,806 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/68001a821d174be48e96a1d2b82eab96 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96 2024-12-03T14:16:21,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733235381787, max timestamp=1733235381808Num uniques 1086; 2024-12-03T14:16:21,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-03T14:16:21,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733235381787, max timestamp=1733235381808Num uniques 1036; 2024-12-03T14:16:21,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733235381787, max timestamp=1733235381808Num uniques 1086; 2024-12-03T14:16:21,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-12-03T14:16:21,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-12-03T14:16:21,815 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into 68001a821d174be48e96a1d2b82eab96(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:21,815 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:21,815 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=13, startTime=1733235381359; duration=0sec 2024-12-03T14:16:21,815 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:21,815 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:21,827 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/23305b7358f1422597db91e79fadc86a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23305b7358f1422597db91e79fadc86a 2024-12-03T14:16:21,833 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 23305b7358f1422597db91e79fadc86a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:21,833 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:21,833 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=13, startTime=1733235381359; duration=0sec 2024-12-03T14:16:21,833 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:21,834 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T14:16:21,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:21,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T14:16:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:21,840 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:21,841 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:21,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733235381832, max timestamp=1733235381852Num uniques 1045; 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733235381832, max timestamp=1733235381852Num uniques 1045; 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733235381832, max timestamp=1733235381852Num uniques 1009; 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13510, suffix heap size=132904,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=295 2024-12-03T14:16:21,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-03T14:16:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235441896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235441895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235441900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235441901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235441901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:22,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235442006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235442006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235442006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235442016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235442016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/e41ffeb3b3764468b191bea8512b5034 2024-12-03T14:16:22,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7c17b731ac341d3bfd087248dd4720d is 50, key is test_row_0/B:col10/1733235380517/Put/seqid=0 2024-12-03T14:16:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742377_1553 (size=12001) 2024-12-03T14:16:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:22,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235442216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235442216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235442216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235442226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235442226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:22,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7c17b731ac341d3bfd087248dd4720d 2024-12-03T14:16:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/0355d5fc003c4acf80db0a91aedbaede is 50, key is test_row_0/C:col10/1733235380517/Put/seqid=0 2024-12-03T14:16:22,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742378_1554 (size=12001) 2024-12-03T14:16:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235442525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235442526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235442526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235442536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235442536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:22,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:22,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/0355d5fc003c4acf80db0a91aedbaede 2024-12-03T14:16:22,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/e41ffeb3b3764468b191bea8512b5034 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034 2024-12-03T14:16:22,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034, entries=150, sequenceid=86, filesize=30.2 K 2024-12-03T14:16:22,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7c17b731ac341d3bfd087248dd4720d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d 2024-12-03T14:16:22,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T14:16:22,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/0355d5fc003c4acf80db0a91aedbaede as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede 2024-12-03T14:16:22,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede, entries=150, sequenceid=86, filesize=11.7 K 2024-12-03T14:16:22,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for c2b13e21c278fc9a8535a3ecbf922878 in 1262ms, sequenceid=86, compaction requested=false 2024-12-03T14:16:22,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:22,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T14:16:22,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:22,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-12-03T14:16:22,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:22,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120332ddb4169f3b49a4b64643f0562a193c_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235381899/Put/seqid=0 2024-12-03T14:16:22,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742379_1555 (size=12304) 2024-12-03T14:16:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:23,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235443040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235443043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235443043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235443046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235443046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235443146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235443156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235443157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:23,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120332ddb4169f3b49a4b64643f0562a193c_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120332ddb4169f3b49a4b64643f0562a193c_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b080a8f6c722440c8e54fa1553034b4a, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:23,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b080a8f6c722440c8e54fa1553034b4a is 175, key is test_row_0/A:col10/1733235381899/Put/seqid=0 2024-12-03T14:16:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742380_1556 (size=31105) 2024-12-03T14:16:23,338 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b080a8f6c722440c8e54fa1553034b4a 2024-12-03T14:16:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e6f62181a77c4ecda4884b93922870cf is 50, key is test_row_0/B:col10/1733235381899/Put/seqid=0 2024-12-03T14:16:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742381_1557 (size=12151) 2024-12-03T14:16:23,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e6f62181a77c4ecda4884b93922870cf 2024-12-03T14:16:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/de28db17e47e40289142bd9c415a1f0b is 50, key is test_row_0/C:col10/1733235381899/Put/seqid=0 2024-12-03T14:16:23,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742382_1558 (size=12151) 2024-12-03T14:16:23,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/de28db17e47e40289142bd9c415a1f0b 2024-12-03T14:16:23,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b080a8f6c722440c8e54fa1553034b4a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a 2024-12-03T14:16:23,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235443356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,361 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a, entries=150, sequenceid=194, filesize=30.4 K 2024-12-03T14:16:23,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e6f62181a77c4ecda4884b93922870cf as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf 2024-12-03T14:16:23,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T14:16:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/de28db17e47e40289142bd9c415a1f0b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b 2024-12-03T14:16:23,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,367 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T14:16:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235443366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235443366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,368 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=33.54 KB/34350 for c2b13e21c278fc9a8535a3ecbf922878 in 446ms, sequenceid=194, compaction requested=true 2024-12-03T14:16:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T14:16:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T14:16:23,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T14:16:23,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-12-03T14:16:23,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5350 sec 2024-12-03T14:16:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:23,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:23,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120316a30e0a478443d5b4e0cb8f32f3bba7_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742383_1559 (size=14794) 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235383686, max timestamp=1733235383702Num uniques 859; 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235383686, max timestamp=1733235383702Num uniques 859; 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733235383686, max timestamp=1733235383702Num uniques 859; 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:16:23,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T14:16:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235443729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235443730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235443731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235443836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235443846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235443846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T14:16:23,986 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:23,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T14:16:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:23,988 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:23,988 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:23,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:24,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235444046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235444056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235444056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235444056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,060 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235444066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,067 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120316a30e0a478443d5b4e0cb8f32f3bba7_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316a30e0a478443d5b4e0cb8f32f3bba7_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:24,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/15df76e71ad94da8993e0a46b4f5cd9e, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:24,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/15df76e71ad94da8993e0a46b4f5cd9e is 175, key is test_row_0/A:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:24,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742384_1560 (size=39749) 2024-12-03T14:16:24,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/15df76e71ad94da8993e0a46b4f5cd9e 2024-12-03T14:16:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:24,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4e89e4d8282c45eba03771b93de9ccd7 is 50, key is test_row_0/B:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:24,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742385_1561 (size=12151) 2024-12-03T14:16:24,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235444356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235444366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235444366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4e89e4d8282c45eba03771b93de9ccd7 2024-12-03T14:16:24,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8921689b11474e8182b888b73abaa17f is 50, key is test_row_0/C:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:24,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742386_1562 (size=12151) 2024-12-03T14:16:24,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:24,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235444866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235444876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235444877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:24,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:24,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8921689b11474e8182b888b73abaa17f 2024-12-03T14:16:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/15df76e71ad94da8993e0a46b4f5cd9e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e 2024-12-03T14:16:24,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e, entries=200, sequenceid=204, filesize=38.8 K 2024-12-03T14:16:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4e89e4d8282c45eba03771b93de9ccd7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7 2024-12-03T14:16:24,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T14:16:24,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8921689b11474e8182b888b73abaa17f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f 2024-12-03T14:16:24,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T14:16:24,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c2b13e21c278fc9a8535a3ecbf922878 in 1263ms, sequenceid=204, compaction requested=true 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:24,932 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:24,932 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:24,933 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132867 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:24,933 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:24,933 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:24,933 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:24,933 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,933 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,933 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/09a0eaa0680d46a494708874787808aa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=47.3 K 2024-12-03T14:16:24,933 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=129.8 K 2024-12-03T14:16:24,933 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:24,933 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e] 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68001a821d174be48e96a1d2b82eab96, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a0eaa0680d46a494708874787808aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c17b731ac341d3bfd087248dd4720d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235380512 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41ffeb3b3764468b191bea8512b5034, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235380512 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f62181a77c4ecda4884b93922870cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733235381898 2024-12-03T14:16:24,934 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b080a8f6c722440c8e54fa1553034b4a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733235381898 2024-12-03T14:16:24,935 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e89e4d8282c45eba03771b93de9ccd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:24,935 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15df76e71ad94da8993e0a46b4f5cd9e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:24,939 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:24,940 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:24,940 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412039949d8d5e62b423cbab11878f320d978_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:24,941 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/a9872b476a4b496e9c3e9824846de0fd is 50, key is test_row_0/B:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:24,942 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412039949d8d5e62b423cbab11878f320d978_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:24,942 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039949d8d5e62b423cbab11878f320d978_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:24,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742387_1563 (size=12391) 2024-12-03T14:16:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742388_1564 (size=4469) 2024-12-03T14:16:25,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T14:16:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:25,057 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T14:16:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:25,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120350c8b2a4d9014f68828120967f06c18e_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235383728/Put/seqid=0 2024-12-03T14:16:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742389_1565 (size=12304) 2024-12-03T14:16:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:25,352 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/a9872b476a4b496e9c3e9824846de0fd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a9872b476a4b496e9c3e9824846de0fd 2024-12-03T14:16:25,355 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into a9872b476a4b496e9c3e9824846de0fd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:25,355 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:25,356 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=12, startTime=1733235384932; duration=0sec 2024-12-03T14:16:25,356 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:25,356 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:25,356 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:16:25,357 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#477 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:25,357 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T14:16:25,357 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:25,357 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:25,357 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23305b7358f1422597db91e79fadc86a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=47.3 K 2024-12-03T14:16:25,357 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/853aaa0653e24048829e4f9afd9bf9a0 is 175, key is test_row_0/A:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:25,358 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 23305b7358f1422597db91e79fadc86a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733235379868 2024-12-03T14:16:25,358 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0355d5fc003c4acf80db0a91aedbaede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733235380512 2024-12-03T14:16:25,358 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting de28db17e47e40289142bd9c415a1f0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733235381898 2024-12-03T14:16:25,358 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 8921689b11474e8182b888b73abaa17f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:25,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742390_1566 (size=31345) 2024-12-03T14:16:25,363 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/853aaa0653e24048829e4f9afd9bf9a0 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0 2024-12-03T14:16:25,365 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:25,365 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/23e0dd920e7d4a0e8648db0d1c5564e5 is 50, key is test_row_0/C:col10/1733235383668/Put/seqid=0 2024-12-03T14:16:25,368 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into 853aaa0653e24048829e4f9afd9bf9a0(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:25,368 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:25,368 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=12, startTime=1733235384932; duration=0sec 2024-12-03T14:16:25,368 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:25,368 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742391_1567 (size=12391) 2024-12-03T14:16:25,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120350c8b2a4d9014f68828120967f06c18e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350c8b2a4d9014f68828120967f06c18e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:25,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9a475810bc6e44a9bc79c30bf327c639, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:25,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9a475810bc6e44a9bc79c30bf327c639 is 175, key is test_row_0/A:col10/1733235383728/Put/seqid=0 2024-12-03T14:16:25,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742392_1568 (size=31105) 2024-12-03T14:16:25,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9a475810bc6e44a9bc79c30bf327c639 2024-12-03T14:16:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/209ded2b96214bc080a5dacc3e09d269 is 50, key is test_row_0/B:col10/1733235383728/Put/seqid=0 2024-12-03T14:16:25,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742393_1569 (size=12151) 2024-12-03T14:16:25,775 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/23e0dd920e7d4a0e8648db0d1c5564e5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23e0dd920e7d4a0e8648db0d1c5564e5 2024-12-03T14:16:25,780 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 23e0dd920e7d4a0e8648db0d1c5564e5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:25,780 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:25,782 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=12, startTime=1733235384932; duration=0sec 2024-12-03T14:16:25,782 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:25,782 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235445892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:25,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235445893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235445894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:25,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:25,919 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/209ded2b96214bc080a5dacc3e09d269 2024-12-03T14:16:25,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/4d17500e06104f0ea99223cd4c396908 is 50, key is test_row_0/C:col10/1733235383728/Put/seqid=0 2024-12-03T14:16:25,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742394_1570 (size=12151) 2024-12-03T14:16:25,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/4d17500e06104f0ea99223cd4c396908 2024-12-03T14:16:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9a475810bc6e44a9bc79c30bf327c639 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639 2024-12-03T14:16:25,982 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639, entries=150, sequenceid=240, filesize=30.4 K 2024-12-03T14:16:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/209ded2b96214bc080a5dacc3e09d269 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269 2024-12-03T14:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,989 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269, entries=150, sequenceid=240, filesize=11.9 K 2024-12-03T14:16:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/4d17500e06104f0ea99223cd4c396908 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908 2024-12-03T14:16:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908, entries=150, sequenceid=240, filesize=11.9 K 2024-12-03T14:16:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for c2b13e21c278fc9a8535a3ecbf922878 in 937ms, sequenceid=240, compaction requested=false 2024-12-03T14:16:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T14:16:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T14:16:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T14:16:25,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0080 sec 2024-12-03T14:16:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0120 sec 2024-12-03T14:16:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:26,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bca7eecedf484082b5c7083beb6a4f7a_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235386039, max timestamp=1733235386050Num uniques 859; 2024-12-03T14:16:26,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235386039, max timestamp=1733235386050Num uniques 859; 2024-12-03T14:16:26,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:26,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:26,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742395_1571 (size=12304) 2024-12-03T14:16:26,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733235386039, max timestamp=1733235386050Num uniques 859; 2024-12-03T14:16:26,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3920e6a7c3b5:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T14:16:26,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bca7eecedf484082b5c7083beb6a4f7a_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bca7eecedf484082b5c7083beb6a4f7a_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:26,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/992ead8dabac4d05be6810b82eb96314, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/992ead8dabac4d05be6810b82eb96314 is 175, key is test_row_0/A:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235446082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742396_1572 (size=31101) 2024-12-03T14:16:26,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/992ead8dabac4d05be6810b82eb96314 2024-12-03T14:16:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235446085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235446085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235446086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235446096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,100 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/86edf4e2724c498389da87b0842b218c is 50, key is test_row_0/B:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T14:16:26,137 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:26,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-03T14:16:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T14:16:26,140 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:26,140 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:26,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:26,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742397_1573 (size=9757) 2024-12-03T14:16:26,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/86edf4e2724c498389da87b0842b218c 2024-12-03T14:16:26,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f591db31aaf84938abcae6cc0d6b41cd is 50, key is test_row_0/C:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742398_1574 (size=9757) 2024-12-03T14:16:26,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235446196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235446196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235446197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235446197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T14:16:26,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T14:16:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235446406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235446406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235446406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235446417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T14:16:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T14:16:26,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f591db31aaf84938abcae6cc0d6b41cd 2024-12-03T14:16:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/992ead8dabac4d05be6810b82eb96314 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314 2024-12-03T14:16:26,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314, entries=150, sequenceid=254, filesize=30.4 K 2024-12-03T14:16:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/86edf4e2724c498389da87b0842b218c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c 2024-12-03T14:16:26,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c, entries=100, sequenceid=254, filesize=9.5 K 2024-12-03T14:16:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f591db31aaf84938abcae6cc0d6b41cd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd 2024-12-03T14:16:26,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd, entries=100, sequenceid=254, filesize=9.5 K 2024-12-03T14:16:26,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c2b13e21c278fc9a8535a3ecbf922878 in 597ms, sequenceid=254, compaction requested=true 2024-12-03T14:16:26,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:26,599 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:26,599 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:26,599 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:26,600 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:26,600 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:26,600 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:26,600 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,600 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,600 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=91.4 K 2024-12-03T14:16:26,600 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a9872b476a4b496e9c3e9824846de0fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=33.5 K 2024-12-03T14:16:26,600 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,600 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314] 2024-12-03T14:16:26,600 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a9872b476a4b496e9c3e9824846de0fd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:26,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T14:16:26,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:26,601 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 209ded2b96214bc080a5dacc3e09d269, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733235383728 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:26,601 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853aaa0653e24048829e4f9afd9bf9a0, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T14:16:26,601 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 86edf4e2724c498389da87b0842b218c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235385892 2024-12-03T14:16:26,601 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a475810bc6e44a9bc79c30bf327c639, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733235383728 2024-12-03T14:16:26,602 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992ead8dabac4d05be6810b82eb96314, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235385890 2024-12-03T14:16:26,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203577401c3a7a3406f8abcfa1ae09018e3_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235386084/Put/seqid=0 2024-12-03T14:16:26,609 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,611 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:26,611 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/a7ed6b8d7b424facadc5f5cc5122fa02 is 50, key is test_row_0/B:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,613 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034ff903ad10d44d3d8c905d5ecff2bca8_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,614 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034ff903ad10d44d3d8c905d5ecff2bca8_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,616 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034ff903ad10d44d3d8c905d5ecff2bca8_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742399_1575 (size=12454) 2024-12-03T14:16:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:26,647 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203577401c3a7a3406f8abcfa1ae09018e3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203577401c3a7a3406f8abcfa1ae09018e3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9648225e9a994fa5a3924a04210e7890, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9648225e9a994fa5a3924a04210e7890 is 175, key is test_row_0/A:col10/1733235386084/Put/seqid=0 2024-12-03T14:16:26,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742400_1576 (size=12493) 2024-12-03T14:16:26,669 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/a7ed6b8d7b424facadc5f5cc5122fa02 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a7ed6b8d7b424facadc5f5cc5122fa02 2024-12-03T14:16:26,675 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into a7ed6b8d7b424facadc5f5cc5122fa02(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:26,675 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:26,676 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=13, startTime=1733235386599; duration=0sec 2024-12-03T14:16:26,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:26,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:26,676 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:26,677 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:26,677 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:26,677 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:26,677 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23e0dd920e7d4a0e8648db0d1c5564e5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=33.5 K 2024-12-03T14:16:26,678 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e0dd920e7d4a0e8648db0d1c5564e5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733235383039 2024-12-03T14:16:26,678 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d17500e06104f0ea99223cd4c396908, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733235383728 2024-12-03T14:16:26,678 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f591db31aaf84938abcae6cc0d6b41cd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235385892 2024-12-03T14:16:26,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742401_1577 (size=4469) 2024-12-03T14:16:26,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:26,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742402_1578 (size=31255) 2024-12-03T14:16:26,742 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:26,742 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/184d6f362e5d4a239c263d3a9caaaa93 is 50, key is test_row_0/C:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235446738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235446738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235446741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235446744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T14:16:26,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742403_1579 (size=12493) 2024-12-03T14:16:26,789 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/184d6f362e5d4a239c263d3a9caaaa93 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/184d6f362e5d4a239c263d3a9caaaa93 2024-12-03T14:16:26,795 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 184d6f362e5d4a239c263d3a9caaaa93(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:26,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:26,796 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=13, startTime=1733235386599; duration=0sec 2024-12-03T14:16:26,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:26,796 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:26,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235446847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235446847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235446856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235446856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235447056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235447056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235447066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235447066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,090 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#487 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:27,091 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b210e9c8c2694b85aba51e9202a2248f is 175, key is test_row_0/A:col10/1733235386000/Put/seqid=0 2024-12-03T14:16:27,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742404_1580 (size=31554) 2024-12-03T14:16:27,103 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/b210e9c8c2694b85aba51e9202a2248f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f 2024-12-03T14:16:27,108 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into b210e9c8c2694b85aba51e9202a2248f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:27,108 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:27,108 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=13, startTime=1733235386599; duration=0sec 2024-12-03T14:16:27,108 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:27,109 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:27,133 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9648225e9a994fa5a3924a04210e7890 2024-12-03T14:16:27,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/577bf42af8cc4ff6beadfe3edaaf6dfd is 50, key is test_row_0/B:col10/1733235386084/Put/seqid=0 2024-12-03T14:16:27,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742405_1581 (size=12301) 2024-12-03T14:16:27,172 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/577bf42af8cc4ff6beadfe3edaaf6dfd 2024-12-03T14:16:27,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/7db7d143ec714b4a8323cabf4f529311 is 50, key is test_row_0/C:col10/1733235386084/Put/seqid=0 2024-12-03T14:16:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742406_1582 (size=12301) 2024-12-03T14:16:27,210 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/7db7d143ec714b4a8323cabf4f529311 2024-12-03T14:16:27,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9648225e9a994fa5a3924a04210e7890 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890 2024-12-03T14:16:27,217 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890, entries=150, sequenceid=290, filesize=30.5 K 2024-12-03T14:16:27,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/577bf42af8cc4ff6beadfe3edaaf6dfd as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd 2024-12-03T14:16:27,221 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T14:16:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/7db7d143ec714b4a8323cabf4f529311 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311 2024-12-03T14:16:27,225 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T14:16:27,226 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c2b13e21c278fc9a8535a3ecbf922878 in 625ms, sequenceid=290, compaction requested=false 2024-12-03T14:16:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T14:16:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T14:16:27,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T14:16:27,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-12-03T14:16:27,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0920 sec 2024-12-03T14:16:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T14:16:27,276 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:27,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-03T14:16:27,279 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:27,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:27,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:27,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T14:16:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:27,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e008d810b00447d98d6ccbc2fde9122_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:27,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742407_1583 (size=12454) 2024-12-03T14:16:27,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:27,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235447392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235447397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235447397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235447397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036e008d810b00447d98d6ccbc2fde9122_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e008d810b00447d98d6ccbc2fde9122_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:27,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/2796b5e977714dae9027c7e3bef021e6, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:27,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/2796b5e977714dae9027c7e3bef021e6 is 175, key is test_row_0/A:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:27,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742408_1584 (size=31255) 2024-12-03T14:16:27,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/2796b5e977714dae9027c7e3bef021e6 2024-12-03T14:16:27,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/cf2de2755a484ebf94223315f38cf53a is 50, key is test_row_0/B:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:27,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742409_1585 (size=12301) 2024-12-03T14:16:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235447506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235447506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:27,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235447716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235447716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235447716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235447716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:27,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/cf2de2755a484ebf94223315f38cf53a 2024-12-03T14:16:27,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/380bf423bb104071888da49304be23d4 is 50, key is test_row_0/C:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:27,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:27,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:27,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742410_1586 (size=12301) 2024-12-03T14:16:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235448026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235448026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235448027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235448027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:28,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/380bf423bb104071888da49304be23d4 2024-12-03T14:16:28,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/2796b5e977714dae9027c7e3bef021e6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6 2024-12-03T14:16:28,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6, entries=150, sequenceid=306, filesize=30.5 K 2024-12-03T14:16:28,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/cf2de2755a484ebf94223315f38cf53a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a 2024-12-03T14:16:28,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T14:16:28,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/380bf423bb104071888da49304be23d4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4 2024-12-03T14:16:28,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4, entries=150, sequenceid=306, filesize=12.0 K 2024-12-03T14:16:28,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c2b13e21c278fc9a8535a3ecbf922878 in 954ms, sequenceid=306, compaction requested=true 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:28,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94064 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:28,322 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:28,323 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,323 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,323 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a7ed6b8d7b424facadc5f5cc5122fa02, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.2 K 2024-12-03T14:16:28,323 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=91.9 K 2024-12-03T14:16:28,323 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6] 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ed6b8d7b424facadc5f5cc5122fa02, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235383730 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting b210e9c8c2694b85aba51e9202a2248f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235383730 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 577bf42af8cc4ff6beadfe3edaaf6dfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733235386073 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9648225e9a994fa5a3924a04210e7890, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733235386073 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2de2755a484ebf94223315f38cf53a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:28,323 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2796b5e977714dae9027c7e3bef021e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:28,327 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:28,329 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412032a5f334f619d40388ce1bba017ad3ae4_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:28,329 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:28,329 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/0e983a014ca4430d812ac9e1ba460518 is 50, key is test_row_0/B:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:28,330 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412032a5f334f619d40388ce1bba017ad3ae4_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:28,330 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032a5f334f619d40388ce1bba017ad3ae4_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:28,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742411_1587 (size=12745) 2024-12-03T14:16:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742412_1588 (size=4469) 2024-12-03T14:16:28,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T14:16:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,350 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:28,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:28,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aef52fc083a84d5cb5a7dcf4d58a15bd_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235387395/Put/seqid=0 2024-12-03T14:16:28,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742413_1589 (size=12454) 2024-12-03T14:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:28,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235448547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235448550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235448551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235448551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235448656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235448656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235448656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235448666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,735 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:28,735 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/169fb5c4e8f84215932f2134f8165bcc is 175, key is test_row_0/A:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:28,738 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/0e983a014ca4430d812ac9e1ba460518 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/0e983a014ca4430d812ac9e1ba460518 2024-12-03T14:16:28,749 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into 0e983a014ca4430d812ac9e1ba460518(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:28,749 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:28,749 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=13, startTime=1733235388322; duration=0sec 2024-12-03T14:16:28,749 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:28,749 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:28,749 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:28,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742414_1590 (size=31699) 2024-12-03T14:16:28,751 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:28,751 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:28,751 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:28,752 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/184d6f362e5d4a239c263d3a9caaaa93, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.2 K 2024-12-03T14:16:28,752 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 184d6f362e5d4a239c263d3a9caaaa93, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733235383730 2024-12-03T14:16:28,752 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db7d143ec714b4a8323cabf4f529311, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733235386073 2024-12-03T14:16:28,752 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 380bf423bb104071888da49304be23d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:28,755 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/169fb5c4e8f84215932f2134f8165bcc as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc 2024-12-03T14:16:28,761 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:28,762 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/c61f4aad266a4bf88ff7f3e800193499 is 50, key is test_row_0/C:col10/1733235387366/Put/seqid=0 2024-12-03T14:16:28,764 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into 169fb5c4e8f84215932f2134f8165bcc(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:28,764 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:28,764 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=13, startTime=1733235388322; duration=0sec 2024-12-03T14:16:28,764 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:28,764 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:28,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203aef52fc083a84d5cb5a7dcf4d58a15bd_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aef52fc083a84d5cb5a7dcf4d58a15bd_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9e34767b4a6441b6bbcd17f0182a44e4, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9e34767b4a6441b6bbcd17f0182a44e4 is 175, key is test_row_0/A:col10/1733235387395/Put/seqid=0 2024-12-03T14:16:28,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742415_1591 (size=12745) 2024-12-03T14:16:28,795 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/c61f4aad266a4bf88ff7f3e800193499 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/c61f4aad266a4bf88ff7f3e800193499 2024-12-03T14:16:28,802 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into c61f4aad266a4bf88ff7f3e800193499(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:28,802 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:28,802 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=13, startTime=1733235388322; duration=0sec 2024-12-03T14:16:28,802 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:28,802 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:28,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742416_1592 (size=31255) 2024-12-03T14:16:28,814 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9e34767b4a6441b6bbcd17f0182a44e4 2024-12-03T14:16:28,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/acea47eb51294015aba9081004bb3f5b is 50, key is test_row_0/B:col10/1733235387395/Put/seqid=0 2024-12-03T14:16:28,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742417_1593 (size=12301) 2024-12-03T14:16:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235448866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235448866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235448867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:28,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235448876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235449187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235449187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235449187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235449187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,258 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/acea47eb51294015aba9081004bb3f5b 2024-12-03T14:16:29,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/19c57b11cb624f41b62415c2a9686494 is 50, key is test_row_0/C:col10/1733235387395/Put/seqid=0 2024-12-03T14:16:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742418_1594 (size=12301) 2024-12-03T14:16:29,311 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/19c57b11cb624f41b62415c2a9686494 2024-12-03T14:16:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9e34767b4a6441b6bbcd17f0182a44e4 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4 2024-12-03T14:16:29,322 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4, entries=150, sequenceid=325, filesize=30.5 K 2024-12-03T14:16:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/acea47eb51294015aba9081004bb3f5b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b 2024-12-03T14:16:29,328 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b, entries=150, sequenceid=325, filesize=12.0 K 2024-12-03T14:16:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/19c57b11cb624f41b62415c2a9686494 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494 2024-12-03T14:16:29,335 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494, entries=150, sequenceid=325, filesize=12.0 K 2024-12-03T14:16:29,337 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c2b13e21c278fc9a8535a3ecbf922878 in 987ms, sequenceid=325, compaction requested=false 2024-12-03T14:16:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-03T14:16:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-03T14:16:29,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T14:16:29,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-12-03T14:16:29,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0640 sec 2024-12-03T14:16:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T14:16:29,427 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:29,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-03T14:16:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:29,429 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:29,430 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:29,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:29,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T14:16:29,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:29,582 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:29,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203267e326aad87421089dfbaafe955187f_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235388550/Put/seqid=0 2024-12-03T14:16:29,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742419_1595 (size=12454) 2024-12-03T14:16:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:29,626 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203267e326aad87421089dfbaafe955187f_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203267e326aad87421089dfbaafe955187f_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/5c4a85dd6588475ea945f0379417a36c, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/5c4a85dd6588475ea945f0379417a36c is 175, key is test_row_0/A:col10/1733235388550/Put/seqid=0 2024-12-03T14:16:29,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742420_1596 (size=31255) 2024-12-03T14:16:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:29,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235449716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235449717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235449719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235449719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:29,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235449826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235449826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235449826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235449826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,034 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/5c4a85dd6588475ea945f0379417a36c 2024-12-03T14:16:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235450036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235450037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235450037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235450037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e7804236760c430d8554245a57b45694 is 50, key is test_row_0/B:col10/1733235388550/Put/seqid=0 2024-12-03T14:16:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:30,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742421_1597 (size=12301) 2024-12-03T14:16:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33246 deadline: 1733235450136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,139 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_2' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235450346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235450346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235450346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235450357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,482 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e7804236760c430d8554245a57b45694 2024-12-03T14:16:30,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9523c62431da4327bb701ec291d9e415 is 50, key is test_row_0/C:col10/1733235388550/Put/seqid=0 2024-12-03T14:16:30,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742422_1598 (size=12301) 2024-12-03T14:16:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235450856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235450856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235450856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235450866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:30,914 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9523c62431da4327bb701ec291d9e415 2024-12-03T14:16:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/5c4a85dd6588475ea945f0379417a36c as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c 2024-12-03T14:16:30,921 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c, entries=150, sequenceid=342, filesize=30.5 K 2024-12-03T14:16:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e7804236760c430d8554245a57b45694 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694 2024-12-03T14:16:30,924 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T14:16:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/9523c62431da4327bb701ec291d9e415 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415 2024-12-03T14:16:30,930 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T14:16:30,931 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c2b13e21c278fc9a8535a3ecbf922878 in 1349ms, sequenceid=342, compaction requested=true 2024-12-03T14:16:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:30,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-03T14:16:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-03T14:16:30,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T14:16:30,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-12-03T14:16:30,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5070 sec 2024-12-03T14:16:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T14:16:31,576 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:31,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-03T14:16:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T14:16:31,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:31,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:31,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T14:16:31,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:31,732 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203785cfe6a05684290b13fbd44af3c66ac_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235389710/Put/seqid=0 2024-12-03T14:16:31,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742423_1599 (size=12454) 2024-12-03T14:16:31,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:31,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235451882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235451883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:31,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235451887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235451886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:31,891 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T14:16:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235451996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235451996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235451996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:32,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203785cfe6a05684290b13fbd44af3c66ac_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203785cfe6a05684290b13fbd44af3c66ac_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:32,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/94937de94ab6424382247aad0d35e522, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:32,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/94937de94ab6424382247aad0d35e522 is 175, key is test_row_0/A:col10/1733235389710/Put/seqid=0 2024-12-03T14:16:32,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742424_1600 (size=31255) 2024-12-03T14:16:32,162 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/94937de94ab6424382247aad0d35e522 2024-12-03T14:16:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4d0c2c8a5ddb4676b0051c71ee812416 is 50, key is test_row_0/B:col10/1733235389710/Put/seqid=0 2024-12-03T14:16:32,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742425_1601 (size=12301) 2024-12-03T14:16:32,174 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4d0c2c8a5ddb4676b0051c71ee812416 2024-12-03T14:16:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/208ff1070d284721a36367ca999df915 is 50, key is test_row_0/C:col10/1733235389710/Put/seqid=0 2024-12-03T14:16:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742426_1602 (size=12301) 2024-12-03T14:16:32,186 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/208ff1070d284721a36367ca999df915 2024-12-03T14:16:32,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/94937de94ab6424382247aad0d35e522 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522 2024-12-03T14:16:32,194 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522, entries=150, sequenceid=361, filesize=30.5 K 2024-12-03T14:16:32,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/4d0c2c8a5ddb4676b0051c71ee812416 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416 2024-12-03T14:16:32,198 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416, entries=150, sequenceid=361, filesize=12.0 K 2024-12-03T14:16:32,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/208ff1070d284721a36367ca999df915 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915 2024-12-03T14:16:32,202 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915, entries=150, sequenceid=361, filesize=12.0 K 2024-12-03T14:16:32,203 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c2b13e21c278fc9a8535a3ecbf922878 in 471ms, sequenceid=361, compaction requested=true 2024-12-03T14:16:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-03T14:16:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-03T14:16:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T14:16:32,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-03T14:16:32,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 624 msec 2024-12-03T14:16:32,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 630 msec 2024-12-03T14:16:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:32,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:32,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:32,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203306cdb35918848e9abadcb2b59fabbe9_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:32,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742427_1603 (size=14994) 2024-12-03T14:16:32,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:32,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203306cdb35918848e9abadcb2b59fabbe9_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203306cdb35918848e9abadcb2b59fabbe9_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:32,232 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/898061fbdcd040a5b91b38e4c4d8c152, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:32,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/898061fbdcd040a5b91b38e4c4d8c152 is 175, key is test_row_0/A:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:32,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742428_1604 (size=39949) 2024-12-03T14:16:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235452236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235452237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235452239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235452346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235452346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235452356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235452556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235452556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235452566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/898061fbdcd040a5b91b38e4c4d8c152 2024-12-03T14:16:32,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/94a2deed77cb4acbaa41798c84ec3c8f is 50, key is test_row_0/B:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:32,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742429_1605 (size=12301) 2024-12-03T14:16:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T14:16:32,717 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:32,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-03T14:16:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:32,720 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:32,721 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:32,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:32,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235452866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235452866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:32,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T14:16:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:32,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:32,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:32,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235452876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T14:16:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:33,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/94a2deed77cb4acbaa41798c84ec3c8f 2024-12-03T14:16:33,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f0aea2bca7804f5abf1719d1cc59c6d6 is 50, key is test_row_0/C:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:33,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742430_1606 (size=12301) 2024-12-03T14:16:33,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f0aea2bca7804f5abf1719d1cc59c6d6 2024-12-03T14:16:33,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/898061fbdcd040a5b91b38e4c4d8c152 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152 2024-12-03T14:16:33,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152, entries=200, sequenceid=375, filesize=39.0 K 2024-12-03T14:16:33,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/94a2deed77cb4acbaa41798c84ec3c8f as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f 2024-12-03T14:16:33,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f, entries=150, sequenceid=375, filesize=12.0 K 2024-12-03T14:16:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/f0aea2bca7804f5abf1719d1cc59c6d6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6 2024-12-03T14:16:33,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6, entries=150, sequenceid=375, filesize=12.0 K 2024-12-03T14:16:33,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c2b13e21c278fc9a8535a3ecbf922878 in 887ms, sequenceid=375, compaction requested=true 2024-12-03T14:16:33,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:33,097 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:33,097 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:16:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:33,098 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:16:33,098 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:33,098 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,098 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/0e983a014ca4430d812ac9e1ba460518, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=60.5 K 2024-12-03T14:16:33,098 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165413 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:16:33,098 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:33,098 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,098 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=161.5 K 2024-12-03T14:16:33,098 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,099 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152] 2024-12-03T14:16:33,099 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e983a014ca4430d812ac9e1ba460518, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:33,099 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 169fb5c4e8f84215932f2134f8165bcc, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:33,102 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting acea47eb51294015aba9081004bb3f5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733235387389 2024-12-03T14:16:33,102 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e34767b4a6441b6bbcd17f0182a44e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733235387389 2024-12-03T14:16:33,102 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting e7804236760c430d8554245a57b45694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235388539 2024-12-03T14:16:33,108 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4a85dd6588475ea945f0379417a36c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235388539 2024-12-03T14:16:33,108 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0c2c8a5ddb4676b0051c71ee812416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733235389710 2024-12-03T14:16:33,108 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 94a2deed77cb4acbaa41798c84ec3c8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:33,108 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94937de94ab6424382247aad0d35e522, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733235389710 2024-12-03T14:16:33,109 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898061fbdcd040a5b91b38e4c4d8c152, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:33,130 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:33,131 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/f89cf1e1abfb4dc9be6456ea49c1be09 is 50, key is test_row_0/B:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:33,137 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:33,149 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412032320a9053893405da254e656403d86d0_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:33,152 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412032320a9053893405da254e656403d86d0_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:33,152 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032320a9053893405da254e656403d86d0_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:33,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742431_1607 (size=12915) 2024-12-03T14:16:33,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-03T14:16:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,181 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:16:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:33,182 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/f89cf1e1abfb4dc9be6456ea49c1be09 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f89cf1e1abfb4dc9be6456ea49c1be09 2024-12-03T14:16:33,188 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into f89cf1e1abfb4dc9be6456ea49c1be09(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:33,188 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:33,188 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=11, startTime=1733235393097; duration=0sec 2024-12-03T14:16:33,188 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:33,188 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:33,188 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T14:16:33,191 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61949 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T14:16:33,191 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:33,191 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:33,191 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/c61f4aad266a4bf88ff7f3e800193499, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=60.5 K 2024-12-03T14:16:33,191 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c61f4aad266a4bf88ff7f3e800193499, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733235386734 2024-12-03T14:16:33,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c57b11cb624f41b62415c2a9686494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733235387389 2024-12-03T14:16:33,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 9523c62431da4327bb701ec291d9e415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733235388539 2024-12-03T14:16:33,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 208ff1070d284721a36367ca999df915, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1733235389710 2024-12-03T14:16:33,192 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f0aea2bca7804f5abf1719d1cc59c6d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742432_1608 (size=4469) 2024-12-03T14:16:33,199 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#510 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:33,200 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f183b9d4b7994798ae2cf4ed061c3be5 is 175, key is test_row_0/A:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c88a2499b4fe43ceb6c672b31551cbe3_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235392236/Put/seqid=0 2024-12-03T14:16:33,215 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:33,216 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/4b1a16b5fac04aceb6d6ce0678663175 is 50, key is test_row_0/C:col10/1733235392208/Put/seqid=0 2024-12-03T14:16:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742433_1609 (size=31869) 2024-12-03T14:16:33,229 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f183b9d4b7994798ae2cf4ed061c3be5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5 2024-12-03T14:16:33,234 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into f183b9d4b7994798ae2cf4ed061c3be5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:33,234 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:33,234 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=11, startTime=1733235393097; duration=0sec 2024-12-03T14:16:33,234 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:33,234 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:33,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742434_1610 (size=12454) 2024-12-03T14:16:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:33,239 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c88a2499b4fe43ceb6c672b31551cbe3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c88a2499b4fe43ceb6c672b31551cbe3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:33,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f706a07f320f4952a6c14b184a127994, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f706a07f320f4952a6c14b184a127994 is 175, key is test_row_0/A:col10/1733235392236/Put/seqid=0 2024-12-03T14:16:33,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742436_1612 (size=31255) 2024-12-03T14:16:33,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742435_1611 (size=12915) 2024-12-03T14:16:33,262 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/4b1a16b5fac04aceb6d6ce0678663175 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4b1a16b5fac04aceb6d6ce0678663175 2024-12-03T14:16:33,265 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 4b1a16b5fac04aceb6d6ce0678663175(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:33,265 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:33,265 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=11, startTime=1733235393097; duration=0sec 2024-12-03T14:16:33,265 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:33,265 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:33,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235453389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235453390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235453390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235453496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235453496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235453496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,650 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f706a07f320f4952a6c14b184a127994 2024-12-03T14:16:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/79b91d28842b43ed8b567e3af81976fe is 50, key is test_row_0/B:col10/1733235392236/Put/seqid=0 2024-12-03T14:16:33,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742437_1613 (size=12301) 2024-12-03T14:16:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235453706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:33,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235453906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:33,908 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235454016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235454016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235454016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,074 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/79b91d28842b43ed8b567e3af81976fe 2024-12-03T14:16:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/1ec902470403451f9fa803d327bb4129 is 50, key is test_row_0/C:col10/1733235392236/Put/seqid=0 2024-12-03T14:16:34,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742438_1614 (size=12301) 2024-12-03T14:16:34,488 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/1ec902470403451f9fa803d327bb4129 2024-12-03T14:16:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f706a07f320f4952a6c14b184a127994 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994 2024-12-03T14:16:34,495 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994, entries=150, sequenceid=393, filesize=30.5 K 2024-12-03T14:16:34,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/79b91d28842b43ed8b567e3af81976fe as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe 2024-12-03T14:16:34,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe, entries=150, sequenceid=393, filesize=12.0 K 2024-12-03T14:16:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/1ec902470403451f9fa803d327bb4129 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129 2024-12-03T14:16:34,505 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129, entries=150, sequenceid=393, filesize=12.0 K 2024-12-03T14:16:34,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c2b13e21c278fc9a8535a3ecbf922878 in 1325ms, sequenceid=393, compaction requested=false 2024-12-03T14:16:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:34,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-03T14:16:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-03T14:16:34,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-03T14:16:34,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7860 sec 2024-12-03T14:16:34,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7920 sec 2024-12-03T14:16:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:34,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:16:34,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:34,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:34,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120370393585d9514066a51b4a8dcf7e656b_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:34,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742439_1615 (size=14994) 2024-12-03T14:16:34,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235454545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235454547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235454547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235454656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235454656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-03T14:16:34,866 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:34,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-03T14:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:34,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:34,869 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235454866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235454866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235454867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:34,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:34,944 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120370393585d9514066a51b4a8dcf7e656b_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120370393585d9514066a51b4a8dcf7e656b_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:34,945 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/fb7ba83f247942e08e2bbd20ab900b80, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:34,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/fb7ba83f247942e08e2bbd20ab900b80 is 175, key is test_row_0/A:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:34,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742440_1616 (size=39949) 2024-12-03T14:16:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:35,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T14:16:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T14:16:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235455176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235455176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235455177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:35,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T14:16:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:35,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/fb7ba83f247942e08e2bbd20ab900b80 2024-12-03T14:16:35,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/f9da0442756349929c672c49960188af is 50, key is test_row_0/B:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742441_1617 (size=12301) 2024-12-03T14:16:35,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/f9da0442756349929c672c49960188af 2024-12-03T14:16:35,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/d6fba63a84d143a18d42172d3f235ee5 is 50, key is test_row_0/C:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:35,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742442_1618 (size=12301) 2024-12-03T14:16:35,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/d6fba63a84d143a18d42172d3f235ee5 2024-12-03T14:16:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/fb7ba83f247942e08e2bbd20ab900b80 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80 2024-12-03T14:16:35,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80, entries=200, sequenceid=413, filesize=39.0 K 2024-12-03T14:16:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/f9da0442756349929c672c49960188af as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af 2024-12-03T14:16:35,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T14:16:35,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/d6fba63a84d143a18d42172d3f235ee5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5 2024-12-03T14:16:35,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T14:16:35,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c2b13e21c278fc9a8535a3ecbf922878 in 900ms, sequenceid=413, compaction requested=true 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:35,429 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:35,429 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:35,430 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:35,430 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:35,430 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:35,430 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,430 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:35,430 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,430 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=100.7 K 2024-12-03T14:16:35,431 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80] 2024-12-03T14:16:35,431 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f89cf1e1abfb4dc9be6456ea49c1be09, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.6 K 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f183b9d4b7994798ae2cf4ed061c3be5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f89cf1e1abfb4dc9be6456ea49c1be09, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting f706a07f320f4952a6c14b184a127994, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733235392234 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b91d28842b43ed8b567e3af81976fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733235392234 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7ba83f247942e08e2bbd20ab900b80, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:35,431 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting f9da0442756349929c672c49960188af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:35,436 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:35,438 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b6e2145232da4a68a112846f7774811c_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:35,439 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:35,439 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/6bde688e4d5847539be3bacc3cd4cef7 is 50, key is test_row_0/B:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:35,440 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b6e2145232da4a68a112846f7774811c_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:35,440 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6e2145232da4a68a112846f7774811c_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:35,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742443_1619 (size=13017) 2024-12-03T14:16:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742444_1620 (size=4469) 2024-12-03T14:16:35,459 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:16:35,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cf0a655e5b046f6a2d6648f48127978_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235394545/Put/seqid=0 2024-12-03T14:16:35,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742445_1621 (size=12454) 2024-12-03T14:16:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:35,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:35,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235455701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235455704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235455704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235455815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235455815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235455816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:35,850 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/6bde688e4d5847539be3bacc3cd4cef7 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/6bde688e4d5847539be3bacc3cd4cef7 2024-12-03T14:16:35,850 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#518 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:35,851 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9fba6ee289fc4cee8307f60ddf0bbc22 is 175, key is test_row_0/A:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:35,857 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into 6bde688e4d5847539be3bacc3cd4cef7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:35,857 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:35,857 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=13, startTime=1733235395429; duration=0sec 2024-12-03T14:16:35,857 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:35,857 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:35,857 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:35,859 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:35,859 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:35,859 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:35,859 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4b1a16b5fac04aceb6d6ce0678663175, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.6 K 2024-12-03T14:16:35,860 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1a16b5fac04aceb6d6ce0678663175, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733235391881 2024-12-03T14:16:35,860 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec902470403451f9fa803d327bb4129, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1733235392234 2024-12-03T14:16:35,860 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting d6fba63a84d143a18d42172d3f235ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742446_1622 (size=31971) 2024-12-03T14:16:35,878 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:35,879 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/1139447b4ca9476b8c54c8756fa54b30 is 50, key is test_row_0/C:col10/1733235394528/Put/seqid=0 2024-12-03T14:16:35,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742447_1623 (size=13017) 2024-12-03T14:16:35,890 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/1139447b4ca9476b8c54c8756fa54b30 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1139447b4ca9476b8c54c8756fa54b30 2024-12-03T14:16:35,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:35,894 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cf0a655e5b046f6a2d6648f48127978_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cf0a655e5b046f6a2d6648f48127978_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/af4af7669f294e9886a4686e564af930, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:35,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/af4af7669f294e9886a4686e564af930 is 175, key is test_row_0/A:col10/1733235394545/Put/seqid=0 2024-12-03T14:16:35,897 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 1139447b4ca9476b8c54c8756fa54b30(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:35,897 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:35,897 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=13, startTime=1733235395429; duration=0sec 2024-12-03T14:16:35,897 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:35,897 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:35,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742448_1624 (size=31255) 2024-12-03T14:16:35,902 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/af4af7669f294e9886a4686e564af930 2024-12-03T14:16:35,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7470887666140be84defdbe360eb4b6 is 50, key is test_row_0/B:col10/1733235394545/Put/seqid=0 2024-12-03T14:16:35,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742449_1625 (size=12301) 2024-12-03T14:16:35,922 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7470887666140be84defdbe360eb4b6 2024-12-03T14:16:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/196b097a2b144f558d8abc43fed85389 is 50, key is test_row_0/C:col10/1733235394545/Put/seqid=0 2024-12-03T14:16:35,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742450_1626 (size=12301) 2024-12-03T14:16:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:36,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235456026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235456026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235456026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,265 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/9fba6ee289fc4cee8307f60ddf0bbc22 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22 2024-12-03T14:16:36,268 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into 9fba6ee289fc4cee8307f60ddf0bbc22(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:36,268 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:36,268 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=13, startTime=1733235395429; duration=0sec 2024-12-03T14:16:36,268 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:36,268 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235456336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235456337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235456337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/196b097a2b144f558d8abc43fed85389 2024-12-03T14:16:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/af4af7669f294e9886a4686e564af930 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930 2024-12-03T14:16:36,361 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930, entries=150, sequenceid=428, filesize=30.5 K 2024-12-03T14:16:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/c7470887666140be84defdbe360eb4b6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6 2024-12-03T14:16:36,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6, entries=150, sequenceid=428, filesize=12.0 K 2024-12-03T14:16:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/196b097a2b144f558d8abc43fed85389 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389 2024-12-03T14:16:36,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389, entries=150, sequenceid=428, filesize=12.0 K 2024-12-03T14:16:36,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c2b13e21c278fc9a8535a3ecbf922878 in 889ms, sequenceid=428, compaction requested=false 2024-12-03T14:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-03T14:16:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-03T14:16:36,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-03T14:16:36,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-12-03T14:16:36,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5060 sec 2024-12-03T14:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:36,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:36,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120337997eaa1662492e8ccd313fd650dce2_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:36,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742451_1627 (size=14994) 2024-12-03T14:16:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235456859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235456859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235456861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235456966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235456966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:36,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235456967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:36,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T14:16:37,016 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:37,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T14:16:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-03T14:16:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:37,019 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T14:16:37,020 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T14:16:37,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T14:16:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:37,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235457178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235457178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235457178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,256 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:37,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120337997eaa1662492e8ccd313fd650dce2_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337997eaa1662492e8ccd313fd650dce2_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:37,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/07c6d51ae5b94951938a62db88fe60da, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:37,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/07c6d51ae5b94951938a62db88fe60da is 175, key is test_row_0/A:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:37,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:37,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742452_1628 (size=39949) 2024-12-03T14:16:37,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/07c6d51ae5b94951938a62db88fe60da 2024-12-03T14:16:37,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/995c0fd69172470e985106273d5ff2d5 is 50, key is test_row_0/B:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742453_1629 (size=12301) 2024-12-03T14:16:37,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235457486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235457486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235457486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:37,669 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-12-03T14:16:37,669 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:37,669 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,669 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-12-03T14:16:37,669 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:37,669 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,669 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:37,669 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:37,670 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-12-03T14:16:37,670 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:37,670 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,670 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:37,671 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-12-03T14:16:37,671 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:37,671 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,672 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:37,672 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-12-03T14:16:37,672 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:37,672 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,672 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:37,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:37,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/995c0fd69172470e985106273d5ff2d5 2024-12-03T14:16:37,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/338b4e48f12346ca96eb8c3dba19302e is 50, key is test_row_0/C:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:37,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742454_1630 (size=12301) 2024-12-03T14:16:37,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:37,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33294 deadline: 1733235457956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,957 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 3920e6a7c3b5,42177,1733235217010 for 'test_row_0' in c2b13e21c278fc9a8535a3ecbf922878 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T14:16:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33238 deadline: 1733235457996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33266 deadline: 1733235457996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T14:16:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33254 deadline: 1733235457996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 , the old value is region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T14:16:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., hostname=3920e6a7c3b5,42177,1733235217010, seqNum=5 because the exception is null or not the one we care about 2024-12-03T14:16:38,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:38,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/338b4e48f12346ca96eb8c3dba19302e 2024-12-03T14:16:38,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/07c6d51ae5b94951938a62db88fe60da as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da 2024-12-03T14:16:38,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da, entries=200, sequenceid=449, filesize=39.0 K 2024-12-03T14:16:38,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/995c0fd69172470e985106273d5ff2d5 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5 2024-12-03T14:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:16:38,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T14:16:38,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/338b4e48f12346ca96eb8c3dba19302e as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e 2024-12-03T14:16:38,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T14:16:38,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c2b13e21c278fc9a8535a3ecbf922878 in 1399ms, sequenceid=449, compaction requested=true 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T14:16:38,246 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2b13e21c278fc9a8535a3ecbf922878:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T14:16:38,246 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:38,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/A is initiating minor compaction (all files) 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/B is initiating minor compaction (all files) 2024-12-03T14:16:38,247 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/A in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,247 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/B in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,247 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/6bde688e4d5847539be3bacc3cd4cef7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.7 K 2024-12-03T14:16:38,247 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=100.8 K 2024-12-03T14:16:38,247 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. files: [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da] 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bde688e4d5847539be3bacc3cd4cef7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fba6ee289fc4cee8307f60ddf0bbc22, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting c7470887666140be84defdbe360eb4b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235394539 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4af7669f294e9886a4686e564af930, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235394539 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 995c0fd69172470e985106273d5ff2d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733235395702 2024-12-03T14:16:38,247 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07c6d51ae5b94951938a62db88fe60da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733235395697 2024-12-03T14:16:38,251 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:38,252 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#B#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:38,252 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/cfbb2df289044b7f924361d9ec0212e2 is 50, key is test_row_0/B:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:38,252 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033dea3c3aaafe4102b63e5323f99d7af1_c2b13e21c278fc9a8535a3ecbf922878 store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:38,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742455_1631 (size=13119) 2024-12-03T14:16:38,258 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033dea3c3aaafe4102b63e5323f99d7af1_c2b13e21c278fc9a8535a3ecbf922878, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:38,258 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033dea3c3aaafe4102b63e5323f99d7af1_c2b13e21c278fc9a8535a3ecbf922878 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:38,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742456_1632 (size=4469) 2024-12-03T14:16:38,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42177 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-03T14:16:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,393 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:38,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031a040b9a2f7e4f3dbf48da2d86c6a6e4_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235396860/Put/seqid=0 2024-12-03T14:16:38,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742457_1633 (size=12454) 2024-12-03T14:16:38,660 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/cfbb2df289044b7f924361d9ec0212e2 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cfbb2df289044b7f924361d9ec0212e2 2024-12-03T14:16:38,663 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:38,663 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/ad43280d96104e41a9aedf30e58a7356 is 175, key is test_row_0/A:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:38,663 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/B of c2b13e21c278fc9a8535a3ecbf922878 into cfbb2df289044b7f924361d9ec0212e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:38,664 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:38,664 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/B, priority=13, startTime=1733235398246; duration=0sec 2024-12-03T14:16:38,664 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T14:16:38,664 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:B 2024-12-03T14:16:38,664 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T14:16:38,665 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T14:16:38,665 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1541): c2b13e21c278fc9a8535a3ecbf922878/C is initiating minor compaction (all files) 2024-12-03T14:16:38,665 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2b13e21c278fc9a8535a3ecbf922878/C in TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:38,665 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1139447b4ca9476b8c54c8756fa54b30, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e] into tmpdir=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp, totalSize=36.7 K 2024-12-03T14:16:38,665 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 1139447b4ca9476b8c54c8756fa54b30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733235393388 2024-12-03T14:16:38,665 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 196b097a2b144f558d8abc43fed85389, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733235394539 2024-12-03T14:16:38,666 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] compactions.Compactor(225): Compacting 338b4e48f12346ca96eb8c3dba19302e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733235395702 2024-12-03T14:16:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742458_1634 (size=32073) 2024-12-03T14:16:38,671 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2b13e21c278fc9a8535a3ecbf922878#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:16:38,672 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/710e4dcb71c94b59b017c91e118f3b7a is 50, key is test_row_0/C:col10/1733235395703/Put/seqid=0 2024-12-03T14:16:38,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742459_1635 (size=13119) 2024-12-03T14:16:38,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:38,812 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031a040b9a2f7e4f3dbf48da2d86c6a6e4_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031a040b9a2f7e4f3dbf48da2d86c6a6e4_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:38,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/28fefb4a83434ea4bd1fb2336593370d, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:38,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/28fefb4a83434ea4bd1fb2336593370d is 175, key is test_row_0/A:col10/1733235396860/Put/seqid=0 2024-12-03T14:16:38,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742460_1636 (size=31255) 2024-12-03T14:16:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42177 {}] regionserver.HRegion(8855): Flush requested on c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:39,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. as already flushing 2024-12-03T14:16:39,011 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-12-03T14:16:39,011 INFO [Thread-2389 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2389. 2024-12-03T14:16:39,011 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:39,011 DEBUG [Thread-2389 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:39,011 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,011 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,011 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,011 DEBUG [Thread-2389 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:39,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:39,018 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-12-03T14:16:39,018 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:39,018 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,018 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:39,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:39,073 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/ad43280d96104e41a9aedf30e58a7356 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/ad43280d96104e41a9aedf30e58a7356 2024-12-03T14:16:39,078 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/A of c2b13e21c278fc9a8535a3ecbf922878 into ad43280d96104e41a9aedf30e58a7356(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:39,078 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:39,078 INFO [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/A, priority=13, startTime=1733235398246; duration=0sec 2024-12-03T14:16:39,079 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:39,079 DEBUG [RS:0;3920e6a7c3b5:42177-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:A 2024-12-03T14:16:39,080 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/710e4dcb71c94b59b017c91e118f3b7a as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/710e4dcb71c94b59b017c91e118f3b7a 2024-12-03T14:16:39,086 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2b13e21c278fc9a8535a3ecbf922878/C of c2b13e21c278fc9a8535a3ecbf922878 into 710e4dcb71c94b59b017c91e118f3b7a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:16:39,086 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:39,086 INFO [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., storeName=c2b13e21c278fc9a8535a3ecbf922878/C, priority=13, startTime=1733235398246; duration=0sec 2024-12-03T14:16:39,086 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T14:16:39,086 DEBUG [RS:0;3920e6a7c3b5:42177-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2b13e21c278fc9a8535a3ecbf922878:C 2024-12-03T14:16:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:39,218 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/28fefb4a83434ea4bd1fb2336593370d 2024-12-03T14:16:39,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/7747cc7bc28144a09db60dae1333bd56 is 50, key is test_row_0/B:col10/1733235396860/Put/seqid=0 2024-12-03T14:16:39,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742461_1637 (size=12301) 2024-12-03T14:16:39,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/7747cc7bc28144a09db60dae1333bd56 2024-12-03T14:16:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8d4c5d5b008f4577a2a3f5b883637750 is 50, key is test_row_0/C:col10/1733235396860/Put/seqid=0 2024-12-03T14:16:39,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742462_1638 (size=12301) 2024-12-03T14:16:40,039 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8d4c5d5b008f4577a2a3f5b883637750 2024-12-03T14:16:40,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/28fefb4a83434ea4bd1fb2336593370d as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/28fefb4a83434ea4bd1fb2336593370d 2024-12-03T14:16:40,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/28fefb4a83434ea4bd1fb2336593370d, entries=150, sequenceid=463, filesize=30.5 K 2024-12-03T14:16:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/7747cc7bc28144a09db60dae1333bd56 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/7747cc7bc28144a09db60dae1333bd56 2024-12-03T14:16:40,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/7747cc7bc28144a09db60dae1333bd56, entries=150, sequenceid=463, filesize=12.0 K 2024-12-03T14:16:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/8d4c5d5b008f4577a2a3f5b883637750 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8d4c5d5b008f4577a2a3f5b883637750 2024-12-03T14:16:40,059 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8d4c5d5b008f4577a2a3f5b883637750, entries=150, sequenceid=463, filesize=12.0 K 2024-12-03T14:16:40,059 INFO [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for c2b13e21c278fc9a8535a3ecbf922878 in 1666ms, sequenceid=463, compaction requested=false 2024-12-03T14:16:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for c2b13e21c278fc9a8535a3ecbf922878: 2024-12-03T14:16:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3920e6a7c3b5:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-03T14:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-03T14:16:40,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-03T14:16:40,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0410 sec 2024-12-03T14:16:40,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.0460 sec 2024-12-03T14:16:40,198 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-12-03T14:16:40,198 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:40,198 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:40,198 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:40,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-03T14:16:41,176 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:48,018 INFO [Thread-2391 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2391. 2024-12-03T14:16:48,018 DEBUG [Thread-2391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T14:16:48,018 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:48,018 DEBUG [Thread-2391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:48,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4002 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4077 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3949 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3998 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4061 2024-12-03T14:16:48,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T14:16:48,018 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:16:48,018 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:16:48,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:48,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:48,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:48,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T14:16:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:48,022 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235408021"}]},"ts":"1733235408021"} 2024-12-03T14:16:48,023 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T14:16:48,023 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T14:16:48,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T14:16:48,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, UNASSIGN}] 2024-12-03T14:16:48,025 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, UNASSIGN 2024-12-03T14:16:48,026 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=CLOSING, regionLocation=3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:48,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, UNASSIGN because future has completed 2024-12-03T14:16:48,027 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:16:48,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010}] 2024-12-03T14:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:48,180 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing c2b13e21c278fc9a8535a3ecbf922878, disabling compactions & flushes 2024-12-03T14:16:48,180 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. after waiting 0 ms 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:48,180 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing c2b13e21c278fc9a8535a3ecbf922878 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=A 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=B 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c2b13e21c278fc9a8535a3ecbf922878, store=C 2024-12-03T14:16:48,180 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T14:16:48,186 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038047bf86e497470eb13742f62f81cfba_c2b13e21c278fc9a8535a3ecbf922878 is 50, key is test_row_0/A:col10/1733235408017/Put/seqid=0 2024-12-03T14:16:48,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742463_1639 (size=12454) 2024-12-03T14:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:48,591 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:16:48,601 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038047bf86e497470eb13742f62f81cfba_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038047bf86e497470eb13742f62f81cfba_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:48,602 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f1be0516884241cfadff358e58e9610b, store: [table=TestAcidGuarantees family=A region=c2b13e21c278fc9a8535a3ecbf922878] 2024-12-03T14:16:48,602 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f1be0516884241cfadff358e58e9610b is 175, key is test_row_0/A:col10/1733235408017/Put/seqid=0 2024-12-03T14:16:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742464_1640 (size=31255) 2024-12-03T14:16:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:49,007 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f1be0516884241cfadff358e58e9610b 2024-12-03T14:16:49,019 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e603e4bf316f4ca695d3d923e1558aad is 50, key is test_row_0/B:col10/1733235408017/Put/seqid=0 2024-12-03T14:16:49,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742465_1641 (size=12301) 2024-12-03T14:16:49,023 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e603e4bf316f4ca695d3d923e1558aad 2024-12-03T14:16:49,028 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/3ebd64d9213f4a01b68b19bb3c4ecddf is 50, key is test_row_0/C:col10/1733235408017/Put/seqid=0 2024-12-03T14:16:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742466_1642 (size=12301) 2024-12-03T14:16:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:49,433 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/3ebd64d9213f4a01b68b19bb3c4ecddf 2024-12-03T14:16:49,443 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/A/f1be0516884241cfadff358e58e9610b as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f1be0516884241cfadff358e58e9610b 2024-12-03T14:16:49,447 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f1be0516884241cfadff358e58e9610b, entries=150, sequenceid=474, filesize=30.5 K 2024-12-03T14:16:49,447 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/B/e603e4bf316f4ca695d3d923e1558aad as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e603e4bf316f4ca695d3d923e1558aad 2024-12-03T14:16:49,451 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e603e4bf316f4ca695d3d923e1558aad, entries=150, sequenceid=474, filesize=12.0 K 2024-12-03T14:16:49,451 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/.tmp/C/3ebd64d9213f4a01b68b19bb3c4ecddf as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/3ebd64d9213f4a01b68b19bb3c4ecddf 2024-12-03T14:16:49,454 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/3ebd64d9213f4a01b68b19bb3c4ecddf, entries=150, sequenceid=474, filesize=12.0 K 2024-12-03T14:16:49,454 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c2b13e21c278fc9a8535a3ecbf922878 in 1274ms, sequenceid=474, compaction requested=true 2024-12-03T14:16:49,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da] to archive 2024-12-03T14:16:49,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:49,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/3af3a4814d424d9fa429b5dc36ed510f 2024-12-03T14:16:49,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/60165803b62f4b39802323cf28460243 2024-12-03T14:16:49,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/68001a821d174be48e96a1d2b82eab96 2024-12-03T14:16:49,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/df7ea8d7ae844b418b1f13dd276f80d6 2024-12-03T14:16:49,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/e41ffeb3b3764468b191bea8512b5034 2024-12-03T14:16:49,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b080a8f6c722440c8e54fa1553034b4a 2024-12-03T14:16:49,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/15df76e71ad94da8993e0a46b4f5cd9e 2024-12-03T14:16:49,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/853aaa0653e24048829e4f9afd9bf9a0 2024-12-03T14:16:49,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9a475810bc6e44a9bc79c30bf327c639 2024-12-03T14:16:49,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/b210e9c8c2694b85aba51e9202a2248f 2024-12-03T14:16:49,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/992ead8dabac4d05be6810b82eb96314 2024-12-03T14:16:49,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9648225e9a994fa5a3924a04210e7890 2024-12-03T14:16:49,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/169fb5c4e8f84215932f2134f8165bcc 2024-12-03T14:16:49,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/2796b5e977714dae9027c7e3bef021e6 2024-12-03T14:16:49,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9e34767b4a6441b6bbcd17f0182a44e4 2024-12-03T14:16:49,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/5c4a85dd6588475ea945f0379417a36c 2024-12-03T14:16:49,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/94937de94ab6424382247aad0d35e522 2024-12-03T14:16:49,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/898061fbdcd040a5b91b38e4c4d8c152 2024-12-03T14:16:49,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f183b9d4b7994798ae2cf4ed061c3be5 2024-12-03T14:16:49,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f706a07f320f4952a6c14b184a127994 2024-12-03T14:16:49,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/fb7ba83f247942e08e2bbd20ab900b80 2024-12-03T14:16:49,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/9fba6ee289fc4cee8307f60ddf0bbc22 2024-12-03T14:16:49,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/af4af7669f294e9886a4686e564af930 2024-12-03T14:16:49,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/07c6d51ae5b94951938a62db88fe60da 2024-12-03T14:16:49,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/09a0eaa0680d46a494708874787808aa, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a9872b476a4b496e9c3e9824846de0fd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a7ed6b8d7b424facadc5f5cc5122fa02, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/0e983a014ca4430d812ac9e1ba460518, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f89cf1e1abfb4dc9be6456ea49c1be09, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/6bde688e4d5847539be3bacc3cd4cef7, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5] to archive 2024-12-03T14:16:49,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:49,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/01c9bec249a743ca9ee9b18befee9015 2024-12-03T14:16:49,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/5d71bd5d270f435a9452d5a0687d53d5 2024-12-03T14:16:49,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/09a0eaa0680d46a494708874787808aa to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/09a0eaa0680d46a494708874787808aa 2024-12-03T14:16:49,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/22359731871e4095827caa8770a9ed89 2024-12-03T14:16:49,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7c17b731ac341d3bfd087248dd4720d 2024-12-03T14:16:49,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e6f62181a77c4ecda4884b93922870cf 2024-12-03T14:16:49,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a9872b476a4b496e9c3e9824846de0fd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a9872b476a4b496e9c3e9824846de0fd 2024-12-03T14:16:49,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4e89e4d8282c45eba03771b93de9ccd7 2024-12-03T14:16:49,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/209ded2b96214bc080a5dacc3e09d269 2024-12-03T14:16:49,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a7ed6b8d7b424facadc5f5cc5122fa02 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/a7ed6b8d7b424facadc5f5cc5122fa02 2024-12-03T14:16:49,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/86edf4e2724c498389da87b0842b218c 2024-12-03T14:16:49,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/577bf42af8cc4ff6beadfe3edaaf6dfd 2024-12-03T14:16:49,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/0e983a014ca4430d812ac9e1ba460518 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/0e983a014ca4430d812ac9e1ba460518 2024-12-03T14:16:49,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cf2de2755a484ebf94223315f38cf53a 2024-12-03T14:16:49,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/acea47eb51294015aba9081004bb3f5b 2024-12-03T14:16:49,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e7804236760c430d8554245a57b45694 2024-12-03T14:16:49,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/4d0c2c8a5ddb4676b0051c71ee812416 2024-12-03T14:16:49,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f89cf1e1abfb4dc9be6456ea49c1be09 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f89cf1e1abfb4dc9be6456ea49c1be09 2024-12-03T14:16:49,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/94a2deed77cb4acbaa41798c84ec3c8f 2024-12-03T14:16:49,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/79b91d28842b43ed8b567e3af81976fe 2024-12-03T14:16:49,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/6bde688e4d5847539be3bacc3cd4cef7 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/6bde688e4d5847539be3bacc3cd4cef7 2024-12-03T14:16:49,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/f9da0442756349929c672c49960188af 2024-12-03T14:16:49,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/c7470887666140be84defdbe360eb4b6 2024-12-03T14:16:49,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/995c0fd69172470e985106273d5ff2d5 2024-12-03T14:16:49,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23305b7358f1422597db91e79fadc86a, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23e0dd920e7d4a0e8648db0d1c5564e5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/184d6f362e5d4a239c263d3a9caaaa93, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/c61f4aad266a4bf88ff7f3e800193499, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4b1a16b5fac04aceb6d6ce0678663175, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1139447b4ca9476b8c54c8756fa54b30, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e] to archive 2024-12-03T14:16:49,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:16:49,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f9d103383e044ab3bc11a9961e515b45 2024-12-03T14:16:49,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9c7afb33fcb5467da91eac5d6abb1fb3 2024-12-03T14:16:49,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23305b7358f1422597db91e79fadc86a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23305b7358f1422597db91e79fadc86a 2024-12-03T14:16:49,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/75ab42fecebb4ee597bdc663fde95a2b 2024-12-03T14:16:49,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/0355d5fc003c4acf80db0a91aedbaede 2024-12-03T14:16:49,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/de28db17e47e40289142bd9c415a1f0b 2024-12-03T14:16:49,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23e0dd920e7d4a0e8648db0d1c5564e5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/23e0dd920e7d4a0e8648db0d1c5564e5 2024-12-03T14:16:49,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8921689b11474e8182b888b73abaa17f 2024-12-03T14:16:49,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4d17500e06104f0ea99223cd4c396908 2024-12-03T14:16:49,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/184d6f362e5d4a239c263d3a9caaaa93 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/184d6f362e5d4a239c263d3a9caaaa93 2024-12-03T14:16:49,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f591db31aaf84938abcae6cc0d6b41cd 2024-12-03T14:16:49,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/7db7d143ec714b4a8323cabf4f529311 2024-12-03T14:16:49,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/c61f4aad266a4bf88ff7f3e800193499 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/c61f4aad266a4bf88ff7f3e800193499 2024-12-03T14:16:49,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/380bf423bb104071888da49304be23d4 2024-12-03T14:16:49,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/19c57b11cb624f41b62415c2a9686494 2024-12-03T14:16:49,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/9523c62431da4327bb701ec291d9e415 2024-12-03T14:16:49,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/208ff1070d284721a36367ca999df915 2024-12-03T14:16:49,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4b1a16b5fac04aceb6d6ce0678663175 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/4b1a16b5fac04aceb6d6ce0678663175 2024-12-03T14:16:49,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/f0aea2bca7804f5abf1719d1cc59c6d6 2024-12-03T14:16:49,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1ec902470403451f9fa803d327bb4129 2024-12-03T14:16:49,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1139447b4ca9476b8c54c8756fa54b30 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/1139447b4ca9476b8c54c8756fa54b30 2024-12-03T14:16:49,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/d6fba63a84d143a18d42172d3f235ee5 2024-12-03T14:16:49,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/196b097a2b144f558d8abc43fed85389 2024-12-03T14:16:49,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/338b4e48f12346ca96eb8c3dba19302e 2024-12-03T14:16:49,543 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits/477.seqid, newMaxSeqId=477, maxSeqId=4 2024-12-03T14:16:49,543 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. 2024-12-03T14:16:49,544 DEBUG [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for c2b13e21c278fc9a8535a3ecbf922878: Waiting for close lock at 1733235408180Running coprocessor pre-close hooks at 1733235408180Disabling compacts and flushes for region at 1733235408180Disabling writes for close at 1733235408180Obtaining lock to block concurrent updates at 1733235408180Preparing flush snapshotting stores in c2b13e21c278fc9a8535a3ecbf922878 at 1733235408180Finished memstore snapshotting TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733235408180Flushing stores of TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878. at 1733235408181 (+1 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/A: creating writer at 1733235408181Flushing mob file c2b13e21c278fc9a8535a3ecbf922878/A: appending metadata at 1733235408186 (+5 ms)Flushing mob file c2b13e21c278fc9a8535a3ecbf922878/A: closing flushed file at 1733235408186Flushing c2b13e21c278fc9a8535a3ecbf922878/A: appending metadata at 1733235408602 (+416 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/A: closing flushed file at 1733235408602Flushing c2b13e21c278fc9a8535a3ecbf922878/B: creating writer at 1733235409016 (+414 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/B: appending metadata at 1733235409019 (+3 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/B: closing flushed file at 1733235409019Flushing c2b13e21c278fc9a8535a3ecbf922878/C: creating writer at 1733235409026 (+7 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/C: appending metadata at 1733235409028 (+2 ms)Flushing c2b13e21c278fc9a8535a3ecbf922878/C: closing flushed file at 1733235409028Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35d51ff7: reopening flushed file at 1733235409442 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ac6fb03: reopening flushed file at 1733235409447 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e6aa93d: reopening flushed file at 1733235409451 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c2b13e21c278fc9a8535a3ecbf922878 in 1274ms, sequenceid=474, compaction requested=true at 1733235409454 (+3 ms)Writing region close event to WAL at 1733235409538 (+84 ms)Running coprocessor post-close hooks at 1733235409543 (+5 ms)Closed at 1733235409543 2024-12-03T14:16:49,546 INFO [RS_CLOSE_REGION-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:49,547 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=c2b13e21c278fc9a8535a3ecbf922878, regionState=CLOSED 2024-12-03T14:16:49,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 because future has completed 2024-12-03T14:16:49,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-03T14:16:49,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure c2b13e21c278fc9a8535a3ecbf922878, server=3920e6a7c3b5,42177,1733235217010 in 1.5230 sec 2024-12-03T14:16:49,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-03T14:16:49,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c2b13e21c278fc9a8535a3ecbf922878, UNASSIGN in 1.5280 sec 2024-12-03T14:16:49,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-03T14:16:49,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5340 sec 2024-12-03T14:16:49,561 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733235409561"}]},"ts":"1733235409561"} 2024-12-03T14:16:49,563 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T14:16:49,563 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T14:16:49,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5450 sec 2024-12-03T14:16:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-03T14:16:50,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:50,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T14:16:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,171 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T14:16:50,173 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,177 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,181 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C, FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits] 2024-12-03T14:16:50,185 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/28fefb4a83434ea4bd1fb2336593370d to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/28fefb4a83434ea4bd1fb2336593370d 2024-12-03T14:16:50,187 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/ad43280d96104e41a9aedf30e58a7356 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/ad43280d96104e41a9aedf30e58a7356 2024-12-03T14:16:50,189 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f1be0516884241cfadff358e58e9610b to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/A/f1be0516884241cfadff358e58e9610b 2024-12-03T14:16:50,193 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/7747cc7bc28144a09db60dae1333bd56 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/7747cc7bc28144a09db60dae1333bd56 2024-12-03T14:16:50,195 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cfbb2df289044b7f924361d9ec0212e2 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/cfbb2df289044b7f924361d9ec0212e2 2024-12-03T14:16:50,197 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e603e4bf316f4ca695d3d923e1558aad to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/B/e603e4bf316f4ca695d3d923e1558aad 2024-12-03T14:16:50,201 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/3ebd64d9213f4a01b68b19bb3c4ecddf to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/3ebd64d9213f4a01b68b19bb3c4ecddf 2024-12-03T14:16:50,202 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/710e4dcb71c94b59b017c91e118f3b7a to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/710e4dcb71c94b59b017c91e118f3b7a 2024-12-03T14:16:50,204 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8d4c5d5b008f4577a2a3f5b883637750 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/C/8d4c5d5b008f4577a2a3f5b883637750 2024-12-03T14:16:50,208 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits/477.seqid to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878/recovered.edits/477.seqid 2024-12-03T14:16:50,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/default/TestAcidGuarantees/c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,209 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T14:16:50,210 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:16:50,211 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T14:16:50,215 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316a30e0a478443d5b4e0cb8f32f3bba7_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316a30e0a478443d5b4e0cb8f32f3bba7_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,216 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031a040b9a2f7e4f3dbf48da2d86c6a6e4_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031a040b9a2f7e4f3dbf48da2d86c6a6e4_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,218 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203267e326aad87421089dfbaafe955187f_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203267e326aad87421089dfbaafe955187f_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,220 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203306cdb35918848e9abadcb2b59fabbe9_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203306cdb35918848e9abadcb2b59fabbe9_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,221 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120332ddb4169f3b49a4b64643f0562a193c_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120332ddb4169f3b49a4b64643f0562a193c_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,223 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337997eaa1662492e8ccd313fd650dce2_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337997eaa1662492e8ccd313fd650dce2_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,225 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350c8b2a4d9014f68828120967f06c18e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120350c8b2a4d9014f68828120967f06c18e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,227 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203577401c3a7a3406f8abcfa1ae09018e3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203577401c3a7a3406f8abcfa1ae09018e3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,229 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120363ef1a62ff9946139c4b43e92a0ffb8e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120363ef1a62ff9946139c4b43e92a0ffb8e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,231 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e008d810b00447d98d6ccbc2fde9122_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036e008d810b00447d98d6ccbc2fde9122_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,233 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120370393585d9514066a51b4a8dcf7e656b_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120370393585d9514066a51b4a8dcf7e656b_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,236 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203785cfe6a05684290b13fbd44af3c66ac_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203785cfe6a05684290b13fbd44af3c66ac_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,237 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038047bf86e497470eb13742f62f81cfba_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038047bf86e497470eb13742f62f81cfba_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,239 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388ce8393fc094a32a7b8d7f9e5b4228e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120388ce8393fc094a32a7b8d7f9e5b4228e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,241 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cf0a655e5b046f6a2d6648f48127978_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cf0a655e5b046f6a2d6648f48127978_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,243 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aef52fc083a84d5cb5a7dcf4d58a15bd_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203aef52fc083a84d5cb5a7dcf4d58a15bd_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,245 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b10a5a5822514fa4be9135e02a170c6e_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b10a5a5822514fa4be9135e02a170c6e_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,247 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bca7eecedf484082b5c7083beb6a4f7a_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bca7eecedf484082b5c7083beb6a4f7a_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,249 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c88a2499b4fe43ceb6c672b31551cbe3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c88a2499b4fe43ceb6c672b31551cbe3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,251 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb1f85d8a6df4946a2adf6f4056af2e3_c2b13e21c278fc9a8535a3ecbf922878 to hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203cb1f85d8a6df4946a2adf6f4056af2e3_c2b13e21c278fc9a8535a3ecbf922878 2024-12-03T14:16:50,252 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T14:16:50,255 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,258 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T14:16:50,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T14:16:50,263 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,263 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T14:16:50,263 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733235410263"}]},"ts":"9223372036854775807"} 2024-12-03T14:16:50,266 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T14:16:50,266 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c2b13e21c278fc9a8535a3ecbf922878, NAME => 'TestAcidGuarantees,,1733235374513.c2b13e21c278fc9a8535a3ecbf922878.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T14:16:50,266 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T14:16:50,266 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733235410266"}]},"ts":"9223372036854775807"} 2024-12-03T14:16:50,268 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T14:16:50,268 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T14:16:50,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-12-03T14:16:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T14:16:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T14:16:50,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T14:16:50,286 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 251), OpenFileDescriptor=443 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 361) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10586 (was 10026) - AvailableMemoryMB LEAK? - 2024-12-03T14:16:50,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T14:16:50,286 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:16:50,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:16:50,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:50,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:50,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:16:50,287 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T14:16:50,287 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=129225742, stopped=false 2024-12-03T14:16:50,287 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3920e6a7c3b5,42837,1733235216263 2024-12-03T14:16:50,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:16:50,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:16:50,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:16:50,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:16:50,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:16:50,361 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:16:50,361 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:16:50,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:50,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:16:50,362 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:16:50,362 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3920e6a7c3b5,42177,1733235217010' ***** 2024-12-03T14:16:50,363 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T14:16:50,363 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T14:16:50,364 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T14:16:50,364 INFO [RS:0;3920e6a7c3b5:42177 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T14:16:50,364 INFO [RS:0;3920e6a7c3b5:42177 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T14:16:50,364 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(959): stopping server 3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:50,364 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:16:50,364 INFO [RS:0;3920e6a7c3b5:42177 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3920e6a7c3b5:42177. 2024-12-03T14:16:50,365 DEBUG [RS:0;3920e6a7c3b5:42177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:16:50,365 DEBUG [RS:0;3920e6a7c3b5:42177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:50,365 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T14:16:50,365 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T14:16:50,365 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T14:16:50,365 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T14:16:50,367 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T14:16:50,367 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T14:16:50,368 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:16:50,368 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:16:50,368 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:16:50,368 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:16:50,368 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:16:50,368 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:16:50,368 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T14:16:50,383 INFO [regionserver/3920e6a7c3b5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:16:50,389 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/info/5c8e1b73048d43a4a27255af95ee79a6 is 95, key is TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b./info:/1733235250460/DeleteFamily/seqid=0 2024-12-03T14:16:50,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742467_1643 (size=6514) 2024-12-03T14:16:50,393 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/info/5c8e1b73048d43a4a27255af95ee79a6 2024-12-03T14:16:50,409 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/ns/34e773a7aa7348e684577f1f768268b3 is 93, key is TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b./ns:/1733235250416/DeleteFamily/seqid=0 2024-12-03T14:16:50,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742468_1644 (size=5906) 2024-12-03T14:16:50,413 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/ns/34e773a7aa7348e684577f1f768268b3 2024-12-03T14:16:50,431 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/rep_barrier/a493a6c8cdea460d8c61f3f640ed47c3 is 102, key is TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b./rep_barrier:/1733235250416/DeleteFamily/seqid=0 2024-12-03T14:16:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742469_1645 (size=6025) 2024-12-03T14:16:50,568 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:16:50,735 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3920e6a7c3b5:42837 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-12-03T14:16:50,768 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:16:50,835 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/rep_barrier/a493a6c8cdea460d8c61f3f640ed47c3 2024-12-03T14:16:50,852 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/table/8ffb43b0277442d7be4106578ef5ef56 is 96, key is TestAcidGuarantees,,1733235221629.8a9798ce308c36fb733235c6d0f2266b./table:/1733235250416/DeleteFamily/seqid=0 2024-12-03T14:16:50,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742470_1646 (size=5936) 2024-12-03T14:16:50,968 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:16:51,092 INFO [regionserver/3920e6a7c3b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T14:16:51,093 INFO [regionserver/3920e6a7c3b5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T14:16:51,168 DEBUG [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:16:51,255 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/table/8ffb43b0277442d7be4106578ef5ef56 2024-12-03T14:16:51,258 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/info/5c8e1b73048d43a4a27255af95ee79a6 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/info/5c8e1b73048d43a4a27255af95ee79a6 2024-12-03T14:16:51,260 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/info/5c8e1b73048d43a4a27255af95ee79a6, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T14:16:51,261 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/ns/34e773a7aa7348e684577f1f768268b3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/ns/34e773a7aa7348e684577f1f768268b3 2024-12-03T14:16:51,263 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/ns/34e773a7aa7348e684577f1f768268b3, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T14:16:51,264 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/rep_barrier/a493a6c8cdea460d8c61f3f640ed47c3 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/rep_barrier/a493a6c8cdea460d8c61f3f640ed47c3 2024-12-03T14:16:51,267 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/rep_barrier/a493a6c8cdea460d8c61f3f640ed47c3, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T14:16:51,267 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/.tmp/table/8ffb43b0277442d7be4106578ef5ef56 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/table/8ffb43b0277442d7be4106578ef5ef56 2024-12-03T14:16:51,271 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/table/8ffb43b0277442d7be4106578ef5ef56, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T14:16:51,272 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 903ms, sequenceid=90, compaction requested=false 2024-12-03T14:16:51,276 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T14:16:51,277 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:16:51,277 INFO [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:16:51,277 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733235410367Running coprocessor pre-close hooks at 1733235410367Disabling compacts and flushes for region at 1733235410367Disabling writes for close at 1733235410368 (+1 ms)Obtaining lock to block concurrent updates at 1733235410368Preparing flush snapshotting stores in 1588230740 at 1733235410368Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733235410368Flushing stores of hbase:meta,,1.1588230740 at 1733235410369 (+1 ms)Flushing 1588230740/info: creating writer at 1733235410369Flushing 1588230740/info: appending metadata at 1733235410389 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733235410389Flushing 1588230740/ns: creating writer at 1733235410396 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733235410409 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733235410409Flushing 1588230740/rep_barrier: creating writer at 1733235410416 (+7 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733235410431 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733235410431Flushing 1588230740/table: creating writer at 1733235410839 (+408 ms)Flushing 1588230740/table: appending metadata at 1733235410851 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733235410851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@717702c0: reopening flushed file at 1733235411258 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab375af: reopening flushed file at 1733235411260 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cf2438: reopening flushed file at 1733235411264 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@614dab78: reopening flushed file at 1733235411267 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 903ms, sequenceid=90, compaction requested=false at 1733235411272 (+5 ms)Writing region close event to WAL at 1733235411274 (+2 ms)Running coprocessor post-close hooks at 1733235411277 (+3 ms)Closed at 1733235411277 2024-12-03T14:16:51,277 DEBUG [RS_CLOSE_META-regionserver/3920e6a7c3b5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T14:16:51,369 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(976): stopping server 3920e6a7c3b5,42177,1733235217010; all regions closed. 2024-12-03T14:16:51,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741834_1010 (size=25344) 2024-12-03T14:16:51,382 DEBUG [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/oldWALs 2024-12-03T14:16:51,382 INFO [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3920e6a7c3b5%2C42177%2C1733235217010.meta:.meta(num 1733235221122) 2024-12-03T14:16:51,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741832_1008 (size=18929792) 2024-12-03T14:16:51,387 DEBUG [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/oldWALs 2024-12-03T14:16:51,387 INFO [RS:0;3920e6a7c3b5:42177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3920e6a7c3b5%2C42177%2C1733235217010:(num 1733235220166) 2024-12-03T14:16:51,387 DEBUG [RS:0;3920e6a7c3b5:42177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:16:51,387 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:16:51,387 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:16:51,388 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.ChoreService(370): Chore service for: regionserver/3920e6a7c3b5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T14:16:51,388 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:16:51,388 INFO [regionserver/3920e6a7c3b5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:16:51,389 INFO [RS:0;3920e6a7c3b5:42177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42177 2024-12-03T14:16:51,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3920e6a7c3b5,42177,1733235217010 2024-12-03T14:16:51,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:16:51,443 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:16:51,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3920e6a7c3b5,42177,1733235217010] 2024-12-03T14:16:51,465 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3920e6a7c3b5,42177,1733235217010 already deleted, retry=false 2024-12-03T14:16:51,465 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3920e6a7c3b5,42177,1733235217010 expired; onlineServers=0 2024-12-03T14:16:51,465 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3920e6a7c3b5,42837,1733235216263' ***** 2024-12-03T14:16:51,466 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:16:51,466 DEBUG [M:0;3920e6a7c3b5:42837 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T14:16:51,466 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T14:16:51,466 DEBUG [M:0;3920e6a7c3b5:42837 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T14:16:51,466 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.small.0-1733235219797 {}] cleaner.HFileCleaner(306): Exit Thread[master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.small.0-1733235219797,5,FailOnTimeoutGroup] 2024-12-03T14:16:51,466 DEBUG [master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.large.0-1733235219794 {}] cleaner.HFileCleaner(306): Exit Thread[master/3920e6a7c3b5:0:becomeActiveMaster-HFileCleaner.large.0-1733235219794,5,FailOnTimeoutGroup] 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.ChoreService(370): Chore service for: master/3920e6a7c3b5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:16:51,466 DEBUG [M:0;3920e6a7c3b5:42837 {}] master.HMaster(1795): Stopping service threads 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T14:16:51,466 INFO [M:0;3920e6a7c3b5:42837 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:16:51,467 ERROR [M:0;3920e6a7c3b5:42837 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1228365231) connection to localhost/127.0.0.1:40805 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40805,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T14:16:51,467 INFO [M:0;3920e6a7c3b5:42837 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T14:16:51,467 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T14:16:51,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T14:16:51,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:16:51,486 DEBUG [M:0;3920e6a7c3b5:42837 {}] zookeeper.ZKUtil(347): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T14:16:51,486 WARN [M:0;3920e6a7c3b5:42837 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T14:16:51,487 INFO [M:0;3920e6a7c3b5:42837 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/.lastflushedseqids 2024-12-03T14:16:51,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742471_1647 (size=72) 2024-12-03T14:16:51,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:16:51,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42177-0x1019b8a88400001, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:16:51,566 INFO [RS:0;3920e6a7c3b5:42177 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:16:51,567 INFO [RS:0;3920e6a7c3b5:42177 {}] regionserver.HRegionServer(1031): Exiting; stopping=3920e6a7c3b5,42177,1733235217010; zookeeper connection closed. 2024-12-03T14:16:51,567 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c56ed96 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c56ed96 2024-12-03T14:16:51,568 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T14:16:51,896 INFO [M:0;3920e6a7c3b5:42837 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T14:16:51,896 INFO [M:0;3920e6a7c3b5:42837 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T14:16:51,897 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:16:51,897 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:16:51,897 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:16:51,897 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:16:51,897 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:16:51,897 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.01 KB heapSize=952.01 KB 2024-12-03T14:16:51,917 DEBUG [M:0;3920e6a7c3b5:42837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/579d9421fb88405aa557d42b504d24f1 is 82, key is hbase:meta,,1/info:regioninfo/1733235221200/Put/seqid=0 2024-12-03T14:16:51,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742472_1648 (size=5672) 2024-12-03T14:16:52,321 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2159 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/579d9421fb88405aa557d42b504d24f1 2024-12-03T14:16:52,345 DEBUG [M:0;3920e6a7c3b5:42837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fcc3d3cf54f4ee1b95ca04dc2a300ea is 2479, key is \x00\x00\x00\x00\x00\x00\x00X/proc:d/1733235313476/Put/seqid=0 2024-12-03T14:16:52,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742473_1649 (size=46040) 2024-12-03T14:16:52,749 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=776.46 KB at sequenceid=2159 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fcc3d3cf54f4ee1b95ca04dc2a300ea 2024-12-03T14:16:52,758 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fcc3d3cf54f4ee1b95ca04dc2a300ea 2024-12-03T14:16:52,778 DEBUG [M:0;3920e6a7c3b5:42837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0585135d40be4d22b5e40540a43e0b08 is 69, key is 3920e6a7c3b5,42177,1733235217010/rs:state/1733235219933/Put/seqid=0 2024-12-03T14:16:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073742474_1650 (size=5156) 2024-12-03T14:16:53,181 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2159 (bloomFilter=true), to=hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0585135d40be4d22b5e40540a43e0b08 2024-12-03T14:16:53,187 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/579d9421fb88405aa557d42b504d24f1 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/579d9421fb88405aa557d42b504d24f1 2024-12-03T14:16:53,192 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/579d9421fb88405aa557d42b504d24f1, entries=8, sequenceid=2159, filesize=5.5 K 2024-12-03T14:16:53,194 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6fcc3d3cf54f4ee1b95ca04dc2a300ea as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fcc3d3cf54f4ee1b95ca04dc2a300ea 2024-12-03T14:16:53,199 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fcc3d3cf54f4ee1b95ca04dc2a300ea 2024-12-03T14:16:53,199 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6fcc3d3cf54f4ee1b95ca04dc2a300ea, entries=174, sequenceid=2159, filesize=45.0 K 2024-12-03T14:16:53,200 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0585135d40be4d22b5e40540a43e0b08 as hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0585135d40be4d22b5e40540a43e0b08 2024-12-03T14:16:53,203 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40805/user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0585135d40be4d22b5e40540a43e0b08, entries=1, sequenceid=2159, filesize=5.0 K 2024-12-03T14:16:53,204 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.01 KB/795663, heapSize ~951.71 KB/974552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2159, compaction requested=false 2024-12-03T14:16:53,205 INFO [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:16:53,205 DEBUG [M:0;3920e6a7c3b5:42837 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733235411896Disabling compacts and flushes for region at 1733235411896Disabling writes for close at 1733235411897 (+1 ms)Obtaining lock to block concurrent updates at 1733235411897Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733235411897Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=795663, getHeapSize=974792, getOffHeapSize=0, getCellsCount=2358 at 1733235411897Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733235411898 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733235411899 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733235411917 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733235411917Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733235412328 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733235412345 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733235412345Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733235412758 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733235412777 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733235412777Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7049f414: reopening flushed file at 1733235413186 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227f0491: reopening flushed file at 1733235413193 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70191d5c: reopening flushed file at 1733235413199 (+6 ms)Finished flush of dataSize ~777.01 KB/795663, heapSize ~951.71 KB/974552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2159, compaction requested=false at 1733235413204 (+5 ms)Writing region close event to WAL at 1733235413205 (+1 ms)Closed at 1733235413205 2024-12-03T14:16:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741830_1006 (size=937213) 2024-12-03T14:16:53,208 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/772f0fb7-f72a-f644-69bf-2ce8f03f881f/MasterData/WALs/3920e6a7c3b5,42837,1733235216263/3920e6a7c3b5%2C42837%2C1733235216263.1733235219119 not finished, retry = 0 2024-12-03T14:16:53,309 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:16:53,309 INFO [M:0;3920e6a7c3b5:42837 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T14:16:53,309 INFO [M:0;3920e6a7c3b5:42837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42837 2024-12-03T14:16:53,310 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:16:53,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:16:53,460 INFO [M:0;3920e6a7c3b5:42837 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:16:53,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42837-0x1019b8a88400000, quorum=127.0.0.1:58521, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:16:53,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:16:53,472 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:16:53,472 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:16:53,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:16:53,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.log.dir/,STOPPED} 2024-12-03T14:16:53,474 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T14:16:53,474 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T14:16:53,474 WARN [BP-1954622350-172.17.0.2-1733235212894 heartbeating to localhost/127.0.0.1:40805 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T14:16:53,474 WARN [BP-1954622350-172.17.0.2-1733235212894 heartbeating to localhost/127.0.0.1:40805 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1954622350-172.17.0.2-1733235212894 (Datanode Uuid 7a205582-7199-4e8f-8cff-3b983e70ba6e) service to localhost/127.0.0.1:40805 2024-12-03T14:16:53,477 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data1/current/BP-1954622350-172.17.0.2-1733235212894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:16:53,477 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/cluster_c7081d26-90a7-cb39-216c-897c2e29e398/data/data2/current/BP-1954622350-172.17.0.2-1733235212894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:16:53,478 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T14:16:53,484 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:16:53,484 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:16:53,484 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:16:53,484 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:16:53,484 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/909229ef-934f-a0c3-7368-f841a7ef7984/hadoop.log.dir/,STOPPED} 2024-12-03T14:16:53,500 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T14:16:53,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down